Snap for 10447354 from 8079887857e71037d3884d47cfecf4bc1584613d to mainline-wifi-release
Change-Id: I8d69a93d23cc2a4aa7adc543d60f902a706cc578
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json
index 92e2ebe..fb46d1e 100644
--- a/.cargo_vcs_info.json
+++ b/.cargo_vcs_info.json
@@ -1,5 +1,6 @@
{
"git": {
- "sha1": "623c09c52c2c38a8d75e94c166593547e8477707"
- }
-}
+ "sha1": "88b1eb54fb66461b9f3524f4b5316241a019279a"
+ },
+ "path_in_vcs": "tokio"
+}
\ No newline at end of file
diff --git a/Android.bp b/Android.bp
index bee1f8a..24cde59 100644
--- a/Android.bp
+++ b/Android.bp
@@ -18,12 +18,11 @@
],
}
-rust_library {
- name: "libtokio",
+rust_defaults {
+ name: "tokio_defaults",
host_supported: true,
- crate_name: "tokio",
cargo_env_compat: true,
- cargo_pkg_version: "1.14.0",
+ cargo_pkg_version: "1.25.0",
srcs: ["src/lib.rs"],
edition: "2018",
features: [
@@ -38,12 +37,12 @@
"num_cpus",
"rt",
"rt-multi-thread",
+ "socket2",
"sync",
"time",
"tokio-macros",
- "winapi",
+ "windows-sys",
],
- cfgs: ["tokio_track_caller"],
rustlibs: [
"libbytes",
"liblibc",
@@ -51,20 +50,37 @@
"libmio",
"libnum_cpus",
"libpin_project_lite",
+ "libsocket2",
],
proc_macros: ["libtokio_macros"],
apex_available: [
"//apex_available:platform",
- "com.android.bluetooth",
+ "com.android.btservices",
"com.android.resolv",
"com.android.uwb",
],
+ product_available: true,
vendor_available: true,
min_sdk_version: "29",
}
+rust_library {
+ name: "libtokio",
+ crate_name: "tokio",
+ defaults: ["tokio_defaults"],
+}
+
+rust_library {
+ name: "libtokio_for_test",
+ crate_name: "tokio",
+ defaults: ["tokio_defaults"],
+ features: [
+ "test-util",
+ ],
+}
+
rust_defaults {
- name: "tokio_defaults_tokio",
+ name: "tokio_defaults_tests",
crate_name: "tokio",
cargo_env_compat: true,
test_suites: ["general-tests"],
@@ -108,7 +124,7 @@
rust_test {
name: "tokio_test_tests__require_full",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/_require_full.rs"],
test_options: {
@@ -118,7 +134,7 @@
rust_test {
name: "tokio_test_tests_buffered",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/buffered.rs"],
test_options: {
@@ -128,7 +144,7 @@
rust_test {
name: "tokio_test_tests_io_async_fd",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_async_fd.rs"],
test_options: {
@@ -138,7 +154,7 @@
rust_test {
name: "tokio_test_tests_io_async_read",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_async_read.rs"],
test_options: {
@@ -148,7 +164,7 @@
rust_test {
name: "tokio_test_tests_io_chain",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_chain.rs"],
test_options: {
@@ -158,7 +174,7 @@
rust_test {
name: "tokio_test_tests_io_copy",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_copy.rs"],
test_options: {
@@ -168,7 +184,7 @@
rust_test {
name: "tokio_test_tests_io_copy_bidirectional",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_copy_bidirectional.rs"],
test_options: {
@@ -178,7 +194,7 @@
rust_test {
name: "tokio_test_tests_io_driver",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_driver.rs"],
test_options: {
@@ -188,7 +204,7 @@
rust_test {
name: "tokio_test_tests_io_driver_drop",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_driver_drop.rs"],
test_options: {
@@ -198,7 +214,7 @@
rust_test {
name: "tokio_test_tests_io_lines",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_lines.rs"],
test_options: {
@@ -208,7 +224,7 @@
rust_test {
name: "tokio_test_tests_io_mem_stream",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_mem_stream.rs"],
test_options: {
@@ -218,7 +234,7 @@
rust_test {
name: "tokio_test_tests_io_read",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read.rs"],
test_options: {
@@ -228,7 +244,7 @@
rust_test {
name: "tokio_test_tests_io_read_buf",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_buf.rs"],
test_options: {
@@ -238,7 +254,7 @@
rust_test {
name: "tokio_test_tests_io_read_exact",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_exact.rs"],
test_options: {
@@ -248,7 +264,7 @@
rust_test {
name: "tokio_test_tests_io_read_line",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_line.rs"],
test_options: {
@@ -258,7 +274,7 @@
rust_test {
name: "tokio_test_tests_io_read_to_end",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_to_end.rs"],
test_options: {
@@ -268,7 +284,7 @@
rust_test {
name: "tokio_test_tests_io_read_to_string",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_to_string.rs"],
test_options: {
@@ -278,7 +294,7 @@
rust_test {
name: "tokio_test_tests_io_read_until",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_until.rs"],
test_options: {
@@ -288,7 +304,7 @@
rust_test {
name: "tokio_test_tests_io_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_split.rs"],
test_options: {
@@ -298,7 +314,7 @@
rust_test {
name: "tokio_test_tests_io_take",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_take.rs"],
test_options: {
@@ -308,7 +324,7 @@
rust_test {
name: "tokio_test_tests_io_write",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write.rs"],
test_options: {
@@ -318,7 +334,7 @@
rust_test {
name: "tokio_test_tests_io_write_all",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write_all.rs"],
test_options: {
@@ -328,7 +344,7 @@
rust_test {
name: "tokio_test_tests_io_write_buf",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write_buf.rs"],
test_options: {
@@ -338,7 +354,7 @@
rust_test {
name: "tokio_test_tests_io_write_int",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write_int.rs"],
test_options: {
@@ -348,7 +364,7 @@
rust_test {
name: "tokio_test_tests_macros_join",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_join.rs"],
test_options: {
@@ -358,7 +374,7 @@
rust_test {
name: "tokio_test_tests_macros_pin",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_pin.rs"],
test_options: {
@@ -368,7 +384,7 @@
rust_test {
name: "tokio_test_tests_macros_select",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_select.rs"],
test_options: {
@@ -378,7 +394,7 @@
rust_test {
name: "tokio_test_tests_macros_test",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_test.rs"],
test_options: {
@@ -388,7 +404,7 @@
rust_test {
name: "tokio_test_tests_macros_try_join",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_try_join.rs"],
test_options: {
@@ -398,7 +414,7 @@
rust_test {
name: "tokio_test_tests_net_bind_resource",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/net_bind_resource.rs"],
test_options: {
@@ -408,7 +424,7 @@
rust_test {
name: "tokio_test_tests_net_lookup_host",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/net_lookup_host.rs"],
test_options: {
@@ -418,7 +434,7 @@
rust_test {
name: "tokio_test_tests_no_rt",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/no_rt.rs"],
test_options: {
@@ -428,7 +444,7 @@
rust_test {
name: "tokio_test_tests_process_kill_on_drop",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/process_kill_on_drop.rs"],
test_options: {
@@ -438,7 +454,7 @@
rust_test {
name: "tokio_test_tests_rt_basic",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/rt_basic.rs"],
test_options: {
@@ -448,7 +464,7 @@
rust_test {
name: "tokio_test_tests_rt_common",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/rt_common.rs"],
test_options: {
@@ -458,7 +474,7 @@
rust_test {
name: "tokio_test_tests_rt_threaded",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/rt_threaded.rs"],
test_options: {
@@ -468,7 +484,7 @@
rust_test {
name: "tokio_test_tests_sync_barrier",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_barrier.rs"],
test_options: {
@@ -478,7 +494,7 @@
rust_test {
name: "tokio_test_tests_sync_broadcast",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_broadcast.rs"],
test_options: {
@@ -488,7 +504,7 @@
rust_test {
name: "tokio_test_tests_sync_errors",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_errors.rs"],
test_options: {
@@ -497,18 +513,8 @@
}
rust_test {
- name: "tokio_test_tests_sync_mpsc",
- defaults: ["tokio_defaults_tokio"],
- host_supported: true,
- srcs: ["tests/sync_mpsc.rs"],
- test_options: {
- unit_test: true,
- },
-}
-
-rust_test {
name: "tokio_test_tests_sync_mutex",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_mutex.rs"],
test_options: {
@@ -518,7 +524,7 @@
rust_test {
name: "tokio_test_tests_sync_mutex_owned",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_mutex_owned.rs"],
test_options: {
@@ -528,7 +534,7 @@
rust_test {
name: "tokio_test_tests_sync_notify",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_notify.rs"],
test_options: {
@@ -538,7 +544,7 @@
rust_test {
name: "tokio_test_tests_sync_oneshot",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_oneshot.rs"],
test_options: {
@@ -548,7 +554,7 @@
rust_test {
name: "tokio_test_tests_sync_rwlock",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_rwlock.rs"],
test_options: {
@@ -558,7 +564,7 @@
rust_test {
name: "tokio_test_tests_sync_semaphore",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_semaphore.rs"],
test_options: {
@@ -568,7 +574,7 @@
rust_test {
name: "tokio_test_tests_sync_semaphore_owned",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_semaphore_owned.rs"],
test_options: {
@@ -578,7 +584,7 @@
rust_test {
name: "tokio_test_tests_sync_watch",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_watch.rs"],
test_options: {
@@ -588,7 +594,7 @@
rust_test {
name: "tokio_test_tests_task_abort",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_abort.rs"],
test_options: {
@@ -598,7 +604,7 @@
rust_test {
name: "tokio_test_tests_task_blocking",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_blocking.rs"],
test_options: {
@@ -608,7 +614,7 @@
rust_test {
name: "tokio_test_tests_task_local",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_local.rs"],
test_options: {
@@ -618,7 +624,7 @@
rust_test {
name: "tokio_test_tests_task_local_set",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_local_set.rs"],
test_options: {
@@ -628,7 +634,7 @@
rust_test {
name: "tokio_test_tests_tcp_accept",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_accept.rs"],
test_options: {
@@ -638,7 +644,7 @@
rust_test {
name: "tokio_test_tests_tcp_connect",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_connect.rs"],
test_options: {
@@ -648,7 +654,7 @@
rust_test {
name: "tokio_test_tests_tcp_echo",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_echo.rs"],
test_options: {
@@ -658,7 +664,7 @@
rust_test {
name: "tokio_test_tests_tcp_into_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_into_split.rs"],
test_options: {
@@ -668,7 +674,7 @@
rust_test {
name: "tokio_test_tests_tcp_into_std",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_into_std.rs"],
test_options: {
@@ -678,7 +684,7 @@
rust_test {
name: "tokio_test_tests_tcp_peek",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_peek.rs"],
test_options: {
@@ -688,7 +694,7 @@
rust_test {
name: "tokio_test_tests_tcp_shutdown",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_shutdown.rs"],
test_options: {
@@ -698,7 +704,7 @@
rust_test {
name: "tokio_test_tests_tcp_socket",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_socket.rs"],
test_options: {
@@ -708,7 +714,7 @@
rust_test {
name: "tokio_test_tests_tcp_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_split.rs"],
test_options: {
@@ -718,7 +724,7 @@
rust_test {
name: "tokio_test_tests_time_rt",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/time_rt.rs"],
test_options: {
@@ -728,7 +734,7 @@
rust_test {
name: "tokio_test_tests_udp",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/udp.rs"],
test_options: {
@@ -738,7 +744,7 @@
rust_test {
name: "tokio_test_tests_uds_cred",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/uds_cred.rs"],
test_options: {
@@ -748,7 +754,7 @@
rust_test {
name: "tokio_test_tests_uds_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/uds_split.rs"],
test_options: {
diff --git a/CHANGELOG.md b/CHANGELOG.md
index afa8bf0..39a57fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,761 @@
+# 1.25.0 (January 28, 2023)
+
+### Fixed
+
+- rt: fix runtime metrics reporting ([#5330])
+
+### Added
+
+- sync: add `broadcast::Sender::len` ([#5343])
+
+### Changed
+
+- fs: increase maximum read buffer size to 2MiB ([#5397])
+
+[#5330]: https://github.com/tokio-rs/tokio/pull/5330
+[#5343]: https://github.com/tokio-rs/tokio/pull/5343
+[#5397]: https://github.com/tokio-rs/tokio/pull/5397
+
+# 1.24.2 (January 17, 2023)
+
+Forward ports 1.18.5 changes.
+
+### Fixed
+
+- io: fix unsoundness in `ReadHalf::unsplit` ([#5375])
+
+[#5375]: https://github.com/tokio-rs/tokio/pull/5375
+
+# 1.24.1 (January 6, 2022)
+
+This release fixes a compilation failure on targets without `AtomicU64` when using rustc older than 1.63. ([#5356])
+
+[#5356]: https://github.com/tokio-rs/tokio/pull/5356
+
+# 1.24.0 (January 5, 2022)
+
+### Fixed
+ - rt: improve native `AtomicU64` support detection ([#5284])
+
+### Added
+ - rt: add configuration option for max number of I/O events polled from the OS
+ per tick ([#5186])
+ - rt: add an environment variable for configuring the default number of worker
+ threads per runtime instance ([#4250])
+
+### Changed
+ - sync: reduce MPSC channel stack usage ([#5294])
+ - io: reduce lock contention in I/O operations ([#5300])
+ - fs: speed up `read_dir()` by chunking operations ([#5309])
+ - rt: use internal `ThreadId` implementation ([#5329])
+ - test: don't auto-advance time when a `spawn_blocking` task is running ([#5115])
+
+[#5186]: https://github.com/tokio-rs/tokio/pull/5186
+[#5294]: https://github.com/tokio-rs/tokio/pull/5294
+[#5284]: https://github.com/tokio-rs/tokio/pull/5284
+[#4250]: https://github.com/tokio-rs/tokio/pull/4250
+[#5300]: https://github.com/tokio-rs/tokio/pull/5300
+[#5329]: https://github.com/tokio-rs/tokio/pull/5329
+[#5115]: https://github.com/tokio-rs/tokio/pull/5115
+[#5309]: https://github.com/tokio-rs/tokio/pull/5309
+
+# 1.23.1 (January 4, 2022)
+
+This release forward ports changes from 1.18.4.
+
+### Fixed
+
+- net: fix Windows named pipe server builder to maintain option when toggling
+ pipe mode ([#5336]).
+
+[#5336]: https://github.com/tokio-rs/tokio/pull/5336
+
+# 1.23.0 (December 5, 2022)
+
+### Fixed
+
+ - net: fix Windows named pipe connect ([#5208])
+ - io: support vectored writes for `ChildStdin` ([#5216])
+ - io: fix `async fn ready()` false positive for OS-specific events ([#5231])
+
+ ### Changed
+ - runtime: `yield_now` defers task until after driver poll ([#5223])
+ - runtime: reduce amount of codegen needed per spawned task ([#5213])
+ - windows: replace `winapi` dependency with `windows-sys` ([#5204])
+
+ [#5208]: https://github.com/tokio-rs/tokio/pull/5208
+ [#5216]: https://github.com/tokio-rs/tokio/pull/5216
+ [#5213]: https://github.com/tokio-rs/tokio/pull/5213
+ [#5204]: https://github.com/tokio-rs/tokio/pull/5204
+ [#5223]: https://github.com/tokio-rs/tokio/pull/5223
+ [#5231]: https://github.com/tokio-rs/tokio/pull/5231
+
+# 1.22.0 (November 17, 2022)
+
+### Added
+ - runtime: add `Handle::runtime_flavor` ([#5138])
+ - sync: add `Mutex::blocking_lock_owned` ([#5130])
+ - sync: add `Semaphore::MAX_PERMITS` ([#5144])
+ - sync: add `merge()` to semaphore permits ([#4948])
+ - sync: add `mpsc::WeakUnboundedSender` ([#5189])
+
+### Added (unstable)
+
+ - process: add `Command::process_group` ([#5114])
+ - runtime: export metrics about the blocking thread pool ([#5161])
+ - task: add `task::id()` and `task::try_id()` ([#5171])
+
+### Fixed
+ - macros: don't take ownership of futures in macros ([#5087])
+ - runtime: fix Stacked Borrows violation in `LocalOwnedTasks` ([#5099])
+ - runtime: mitigate ABA with 32-bit queue indices when possible ([#5042])
+ - task: wake local tasks to the local queue when woken by the same thread ([#5095])
+ - time: panic in release mode when `mark_pending` called illegally ([#5093])
+ - runtime: fix typo in expect message ([#5169])
+ - runtime: fix `unsync_load` on atomic types ([#5175])
+ - task: elaborate safety comments in task deallocation ([#5172])
+ - runtime: fix `LocalSet` drop in thread local ([#5179])
+ - net: remove libc type leakage in a public API ([#5191])
+ - runtime: update the alignment of `CachePadded` ([#5106])
+
+### Changed
+ - io: make `tokio::io::copy` continue filling the buffer when writer stalls ([#5066])
+ - runtime: remove `coop::budget` from `LocalSet::run_until` ([#5155])
+ - sync: make `Notify` panic safe ([#5154])
+
+### Documented
+ - io: fix doc for `write_i8` to use signed integers ([#5040])
+ - net: fix doc typos for TCP and UDP `set_tos` methods ([#5073])
+ - net: fix function name in `UdpSocket::recv` documentation ([#5150])
+ - sync: typo in `TryLockError` for `RwLock::try_write` ([#5160])
+ - task: document that spawned tasks execute immediately ([#5117])
+ - time: document return type of `timeout` ([#5118])
+ - time: document that `timeout` checks only before poll ([#5126])
+ - sync: specify return type of `oneshot::Receiver` in docs ([#5198])
+
+### Internal changes
+ - runtime: use const `Mutex::new` for globals ([#5061])
+ - runtime: remove `Option` around `mio::Events` in io driver ([#5078])
+ - runtime: remove a conditional compilation clause ([#5104])
+ - runtime: remove a reference to internal time handle ([#5107])
+ - runtime: misc time driver cleanup ([#5120])
+ - runtime: move signal driver to runtime module ([#5121])
+ - runtime: signal driver now uses I/O driver directly ([#5125])
+ - runtime: start decoupling I/O driver and I/O handle ([#5127])
+ - runtime: switch `io::handle` refs with scheduler:Handle ([#5128])
+ - runtime: remove Arc from I/O driver ([#5134])
+ - runtime: use signal driver handle via `scheduler::Handle` ([#5135])
+ - runtime: move internal clock fns out of context ([#5139])
+ - runtime: remove `runtime::context` module ([#5140])
+ - runtime: keep driver cfgs in `driver.rs` ([#5141])
+ - runtime: add `runtime::context` to unify thread-locals ([#5143])
+ - runtime: rename some confusing internal variables/fns ([#5151])
+ - runtime: move `coop` mod into `runtime` ([#5152])
+ - runtime: move budget state to context thread-local ([#5157])
+ - runtime: move park logic into runtime module ([#5158])
+ - runtime: move `Runtime` into its own file ([#5159])
+ - runtime: unify entering a runtime with `Handle::enter` ([#5163])
+ - runtime: remove handle reference from each scheduler ([#5166])
+ - runtime: move `enter` into `context` ([#5167])
+ - runtime: combine context and entered thread-locals ([#5168])
+ - runtime: fix accidental unsetting of current handle ([#5178])
+ - runtime: move `CoreStage` methods to `Core` ([#5182])
+ - sync: name mpsc semaphore types ([#5146])
+
+[#4948]: https://github.com/tokio-rs/tokio/pull/4948
+[#5040]: https://github.com/tokio-rs/tokio/pull/5040
+[#5042]: https://github.com/tokio-rs/tokio/pull/5042
+[#5061]: https://github.com/tokio-rs/tokio/pull/5061
+[#5066]: https://github.com/tokio-rs/tokio/pull/5066
+[#5073]: https://github.com/tokio-rs/tokio/pull/5073
+[#5078]: https://github.com/tokio-rs/tokio/pull/5078
+[#5087]: https://github.com/tokio-rs/tokio/pull/5087
+[#5093]: https://github.com/tokio-rs/tokio/pull/5093
+[#5095]: https://github.com/tokio-rs/tokio/pull/5095
+[#5099]: https://github.com/tokio-rs/tokio/pull/5099
+[#5104]: https://github.com/tokio-rs/tokio/pull/5104
+[#5106]: https://github.com/tokio-rs/tokio/pull/5106
+[#5107]: https://github.com/tokio-rs/tokio/pull/5107
+[#5114]: https://github.com/tokio-rs/tokio/pull/5114
+[#5117]: https://github.com/tokio-rs/tokio/pull/5117
+[#5118]: https://github.com/tokio-rs/tokio/pull/5118
+[#5120]: https://github.com/tokio-rs/tokio/pull/5120
+[#5121]: https://github.com/tokio-rs/tokio/pull/5121
+[#5125]: https://github.com/tokio-rs/tokio/pull/5125
+[#5126]: https://github.com/tokio-rs/tokio/pull/5126
+[#5127]: https://github.com/tokio-rs/tokio/pull/5127
+[#5128]: https://github.com/tokio-rs/tokio/pull/5128
+[#5130]: https://github.com/tokio-rs/tokio/pull/5130
+[#5134]: https://github.com/tokio-rs/tokio/pull/5134
+[#5135]: https://github.com/tokio-rs/tokio/pull/5135
+[#5138]: https://github.com/tokio-rs/tokio/pull/5138
+[#5138]: https://github.com/tokio-rs/tokio/pull/5138
+[#5139]: https://github.com/tokio-rs/tokio/pull/5139
+[#5140]: https://github.com/tokio-rs/tokio/pull/5140
+[#5141]: https://github.com/tokio-rs/tokio/pull/5141
+[#5143]: https://github.com/tokio-rs/tokio/pull/5143
+[#5144]: https://github.com/tokio-rs/tokio/pull/5144
+[#5144]: https://github.com/tokio-rs/tokio/pull/5144
+[#5146]: https://github.com/tokio-rs/tokio/pull/5146
+[#5150]: https://github.com/tokio-rs/tokio/pull/5150
+[#5151]: https://github.com/tokio-rs/tokio/pull/5151
+[#5152]: https://github.com/tokio-rs/tokio/pull/5152
+[#5154]: https://github.com/tokio-rs/tokio/pull/5154
+[#5155]: https://github.com/tokio-rs/tokio/pull/5155
+[#5157]: https://github.com/tokio-rs/tokio/pull/5157
+[#5158]: https://github.com/tokio-rs/tokio/pull/5158
+[#5159]: https://github.com/tokio-rs/tokio/pull/5159
+[#5160]: https://github.com/tokio-rs/tokio/pull/5160
+[#5161]: https://github.com/tokio-rs/tokio/pull/5161
+[#5163]: https://github.com/tokio-rs/tokio/pull/5163
+[#5166]: https://github.com/tokio-rs/tokio/pull/5166
+[#5167]: https://github.com/tokio-rs/tokio/pull/5167
+[#5168]: https://github.com/tokio-rs/tokio/pull/5168
+[#5169]: https://github.com/tokio-rs/tokio/pull/5169
+[#5171]: https://github.com/tokio-rs/tokio/pull/5171
+[#5172]: https://github.com/tokio-rs/tokio/pull/5172
+[#5175]: https://github.com/tokio-rs/tokio/pull/5175
+[#5178]: https://github.com/tokio-rs/tokio/pull/5178
+[#5179]: https://github.com/tokio-rs/tokio/pull/5179
+[#5182]: https://github.com/tokio-rs/tokio/pull/5182
+[#5189]: https://github.com/tokio-rs/tokio/pull/5189
+[#5191]: https://github.com/tokio-rs/tokio/pull/5191
+[#5198]: https://github.com/tokio-rs/tokio/pull/5198
+
+# 1.21.2 (September 27, 2022)
+
+This release removes the dependency on the `once_cell` crate to restore the MSRV
+of 1.21.x, which is the latest minor version at the time of release. ([#5048])
+
+[#5048]: https://github.com/tokio-rs/tokio/pull/5048
+
+# 1.21.1 (September 13, 2022)
+
+### Fixed
+
+- net: fix dependency resolution for socket2 ([#5000])
+- task: ignore failure to set TLS in `LocalSet` Drop ([#4976])
+
+[#4976]: https://github.com/tokio-rs/tokio/pull/4976
+[#5000]: https://github.com/tokio-rs/tokio/pull/5000
+
+# 1.21.0 (September 2, 2022)
+
+This release is the first release of Tokio to intentionally support WASM. The
+`sync,macros,io-util,rt,time` features are stabilized on WASM. Additionally the
+wasm32-wasi target is given unstable support for the `net` feature.
+
+### Added
+
+- net: add `device` and `bind_device` methods to TCP/UDP sockets ([#4882])
+- net: add `tos` and `set_tos` methods to TCP and UDP sockets ([#4877])
+- net: add security flags to named pipe `ServerOptions` ([#4845])
+- signal: add more windows signal handlers ([#4924])
+- sync: add `mpsc::Sender::max_capacity` method ([#4904])
+- sync: implement Weak version of `mpsc::Sender` ([#4595])
+- task: add `LocalSet::enter` ([#4765])
+- task: stabilize `JoinSet` and `AbortHandle` ([#4920])
+- tokio: add `track_caller` to public APIs ([#4805], [#4848], [#4852])
+- wasm: initial support for `wasm32-wasi` target ([#4716])
+
+### Fixed
+
+- miri: improve miri compatibility by avoiding temporary references in `linked_list::Link` impls ([#4841])
+- signal: don't register write interest on signal pipe ([#4898])
+- sync: add `#[must_use]` to lock guards ([#4886])
+- sync: fix hang when calling `recv` on closed and reopened broadcast channel ([#4867])
+- task: propagate attributes on task-locals ([#4837])
+
+### Changed
+
+- fs: change panic to error in `File::start_seek` ([#4897])
+- io: reduce syscalls in `poll_read` ([#4840])
+- process: use blocking threadpool for child stdio I/O ([#4824])
+- signal: make `SignalKind` methods const ([#4956])
+
+### Internal changes
+
+- rt: extract `basic_scheduler::Config` ([#4935])
+- rt: move I/O driver into `runtime` module ([#4942])
+- rt: rename internal scheduler types ([#4945])
+
+### Documented
+
+- chore: fix typos and grammar ([#4858], [#4894], [#4928])
+- io: fix typo in `AsyncSeekExt::rewind` docs ([#4893])
+- net: add documentation to `try_read()` for zero-length buffers ([#4937])
+- runtime: remove incorrect panic section for `Builder::worker_threads` ([#4849])
+- sync: doc of `watch::Sender::send` improved ([#4959])
+- task: add cancel safety docs to `JoinHandle` ([#4901])
+- task: expand on cancellation of `spawn_blocking` ([#4811])
+- time: clarify that the first tick of `Interval::tick` happens immediately ([#4951])
+
+### Unstable
+
+- rt: add unstable option to disable the LIFO slot ([#4936])
+- task: fix incorrect signature in `Builder::spawn_on` ([#4953])
+- task: make `task::Builder::spawn*` methods fallible ([#4823])
+
+[#4595]: https://github.com/tokio-rs/tokio/pull/4595
+[#4716]: https://github.com/tokio-rs/tokio/pull/4716
+[#4765]: https://github.com/tokio-rs/tokio/pull/4765
+[#4805]: https://github.com/tokio-rs/tokio/pull/4805
+[#4811]: https://github.com/tokio-rs/tokio/pull/4811
+[#4823]: https://github.com/tokio-rs/tokio/pull/4823
+[#4824]: https://github.com/tokio-rs/tokio/pull/4824
+[#4837]: https://github.com/tokio-rs/tokio/pull/4837
+[#4840]: https://github.com/tokio-rs/tokio/pull/4840
+[#4841]: https://github.com/tokio-rs/tokio/pull/4841
+[#4845]: https://github.com/tokio-rs/tokio/pull/4845
+[#4848]: https://github.com/tokio-rs/tokio/pull/4848
+[#4849]: https://github.com/tokio-rs/tokio/pull/4849
+[#4852]: https://github.com/tokio-rs/tokio/pull/4852
+[#4858]: https://github.com/tokio-rs/tokio/pull/4858
+[#4867]: https://github.com/tokio-rs/tokio/pull/4867
+[#4877]: https://github.com/tokio-rs/tokio/pull/4877
+[#4882]: https://github.com/tokio-rs/tokio/pull/4882
+[#4886]: https://github.com/tokio-rs/tokio/pull/4886
+[#4893]: https://github.com/tokio-rs/tokio/pull/4893
+[#4894]: https://github.com/tokio-rs/tokio/pull/4894
+[#4897]: https://github.com/tokio-rs/tokio/pull/4897
+[#4898]: https://github.com/tokio-rs/tokio/pull/4898
+[#4901]: https://github.com/tokio-rs/tokio/pull/4901
+[#4904]: https://github.com/tokio-rs/tokio/pull/4904
+[#4920]: https://github.com/tokio-rs/tokio/pull/4920
+[#4924]: https://github.com/tokio-rs/tokio/pull/4924
+[#4928]: https://github.com/tokio-rs/tokio/pull/4928
+[#4935]: https://github.com/tokio-rs/tokio/pull/4935
+[#4936]: https://github.com/tokio-rs/tokio/pull/4936
+[#4937]: https://github.com/tokio-rs/tokio/pull/4937
+[#4942]: https://github.com/tokio-rs/tokio/pull/4942
+[#4945]: https://github.com/tokio-rs/tokio/pull/4945
+[#4951]: https://github.com/tokio-rs/tokio/pull/4951
+[#4953]: https://github.com/tokio-rs/tokio/pull/4953
+[#4956]: https://github.com/tokio-rs/tokio/pull/4956
+[#4959]: https://github.com/tokio-rs/tokio/pull/4959
+
+# 1.20.4 (January 17, 2023)
+
+Forward ports 1.18.5 changes.
+
+### Fixed
+
+- io: fix unsoundness in `ReadHalf::unsplit` ([#5375])
+
+[#5375]: https://github.com/tokio-rs/tokio/pull/5375
+
+# 1.20.3 (January 3, 2022)
+
+This release forward ports changes from 1.18.4.
+
+### Fixed
+
+- net: fix Windows named pipe server builder to maintain option when toggling
+ pipe mode ([#5336]).
+
+[#5336]: https://github.com/tokio-rs/tokio/pull/5336
+
+# 1.20.2 (September 27, 2022)
+
+This release removes the dependency on the `once_cell` crate to restore the MSRV
+of the 1.20.x LTS release. ([#5048])
+
+[#5048]: https://github.com/tokio-rs/tokio/pull/5048
+
+# 1.20.1 (July 25, 2022)
+
+### Fixed
+
+- chore: fix version detection in build script ([#4860])
+
+[#4860]: https://github.com/tokio-rs/tokio/pull/4860
+
+# 1.20.0 (July 12, 2022)
+
+### Added
+- tokio: add `track_caller` to public APIs ([#4772], [#4791], [#4793], [#4806], [#4808])
+- sync: Add `has_changed` method to `watch::Ref` ([#4758])
+
+### Changed
+
+- time: remove `src/time/driver/wheel/stack.rs` ([#4766])
+- rt: clean up arguments passed to basic scheduler ([#4767])
+- net: be more specific about winapi features ([#4764])
+- tokio: use const initialized thread locals where possible ([#4677])
+- task: various small improvements to LocalKey ([#4795])
+
+### Documented
+
+- fs: warn about performance pitfall ([#4762])
+- chore: fix spelling ([#4769])
+- sync: document spurious failures in oneshot ([#4777])
+- sync: add warning for watch in non-Send futures ([#4741])
+- chore: fix typo ([#4798])
+
+### Unstable
+
+- joinset: rename `join_one` to `join_next` ([#4755])
+- rt: unhandled panic config for current thread rt ([#4770])
+
+[#4677]: https://github.com/tokio-rs/tokio/pull/4677
+[#4741]: https://github.com/tokio-rs/tokio/pull/4741
+[#4755]: https://github.com/tokio-rs/tokio/pull/4755
+[#4758]: https://github.com/tokio-rs/tokio/pull/4758
+[#4762]: https://github.com/tokio-rs/tokio/pull/4762
+[#4764]: https://github.com/tokio-rs/tokio/pull/4764
+[#4766]: https://github.com/tokio-rs/tokio/pull/4766
+[#4767]: https://github.com/tokio-rs/tokio/pull/4767
+[#4769]: https://github.com/tokio-rs/tokio/pull/4769
+[#4770]: https://github.com/tokio-rs/tokio/pull/4770
+[#4772]: https://github.com/tokio-rs/tokio/pull/4772
+[#4777]: https://github.com/tokio-rs/tokio/pull/4777
+[#4791]: https://github.com/tokio-rs/tokio/pull/4791
+[#4793]: https://github.com/tokio-rs/tokio/pull/4793
+[#4795]: https://github.com/tokio-rs/tokio/pull/4795
+[#4798]: https://github.com/tokio-rs/tokio/pull/4798
+[#4806]: https://github.com/tokio-rs/tokio/pull/4806
+[#4808]: https://github.com/tokio-rs/tokio/pull/4808
+
+# 1.19.2 (June 6, 2022)
+
+This release fixes another bug in `Notified::enable`. ([#4751])
+
+[#4751]: https://github.com/tokio-rs/tokio/pull/4751
+
+# 1.19.1 (June 5, 2022)
+
+This release fixes a bug in `Notified::enable`. ([#4747])
+
+[#4747]: https://github.com/tokio-rs/tokio/pull/4747
+
+# 1.19.0 (June 3, 2022)
+
+### Added
+
+- runtime: add `is_finished` method for `JoinHandle` and `AbortHandle` ([#4709])
+- runtime: make global queue and event polling intervals configurable ([#4671])
+- sync: add `Notified::enable` ([#4705])
+- sync: add `watch::Sender::send_if_modified` ([#4591])
+- sync: add resubscribe method to broadcast::Receiver ([#4607])
+- net: add `take_error` to `TcpSocket` and `TcpStream` ([#4739])
+
+### Changed
+
+- io: refactor out usage of Weak in the io handle ([#4656])
+
+### Fixed
+
+- macros: avoid starvation in `join!` and `try_join!` ([#4624])
+
+### Documented
+
+- runtime: clarify semantics of tasks outliving `block_on` ([#4729])
+- time: fix example for `MissedTickBehavior::Burst` ([#4713])
+
+### Unstable
+
+- metrics: correctly update atomics in `IoDriverMetrics` ([#4725])
+- metrics: fix compilation with unstable, process, and rt, but without net ([#4682])
+- task: add `#[track_caller]` to `JoinSet`/`JoinMap` ([#4697])
+- task: add `Builder::{spawn_on, spawn_local_on, spawn_blocking_on}` ([#4683])
+- task: add `consume_budget` for cooperative scheduling ([#4498])
+- task: add `join_set::Builder` for configuring `JoinSet` tasks ([#4687])
+- task: update return value of `JoinSet::join_one` ([#4726])
+
+[#4498]: https://github.com/tokio-rs/tokio/pull/4498
+[#4591]: https://github.com/tokio-rs/tokio/pull/4591
+[#4607]: https://github.com/tokio-rs/tokio/pull/4607
+[#4624]: https://github.com/tokio-rs/tokio/pull/4624
+[#4656]: https://github.com/tokio-rs/tokio/pull/4656
+[#4671]: https://github.com/tokio-rs/tokio/pull/4671
+[#4682]: https://github.com/tokio-rs/tokio/pull/4682
+[#4683]: https://github.com/tokio-rs/tokio/pull/4683
+[#4687]: https://github.com/tokio-rs/tokio/pull/4687
+[#4697]: https://github.com/tokio-rs/tokio/pull/4697
+[#4705]: https://github.com/tokio-rs/tokio/pull/4705
+[#4709]: https://github.com/tokio-rs/tokio/pull/4709
+[#4713]: https://github.com/tokio-rs/tokio/pull/4713
+[#4725]: https://github.com/tokio-rs/tokio/pull/4725
+[#4726]: https://github.com/tokio-rs/tokio/pull/4726
+[#4729]: https://github.com/tokio-rs/tokio/pull/4729
+[#4739]: https://github.com/tokio-rs/tokio/pull/4739
+
+# 1.18.5 (January 17, 2023)
+
+### Fixed
+
+- io: fix unsoundness in `ReadHalf::unsplit` ([#5375])
+
+[#5375]: https://github.com/tokio-rs/tokio/pull/5375
+
+# 1.18.4 (January 3, 2022)
+
+### Fixed
+
+- net: fix Windows named pipe server builder to maintain option when toggling
+ pipe mode ([#5336]).
+
+[#5336]: https://github.com/tokio-rs/tokio/pull/5336
+
+# 1.18.3 (September 27, 2022)
+
+This release removes the dependency on the `once_cell` crate to restore the MSRV
+of the 1.18.x LTS release. ([#5048])
+
+[#5048]: https://github.com/tokio-rs/tokio/pull/5048
+
+# 1.18.2 (May 5, 2022)
+
+Add missing features for the `winapi` dependency. ([#4663])
+
+[#4663]: https://github.com/tokio-rs/tokio/pull/4663
+
+# 1.18.1 (May 2, 2022)
+
+The 1.18.0 release broke the build for targets without 64-bit atomics when
+building with `tokio_unstable`. This release fixes that. ([#4649])
+
+[#4649]: https://github.com/tokio-rs/tokio/pull/4649
+
+# 1.18.0 (April 27, 2022)
+
+This release adds a number of new APIs in `tokio::net`, `tokio::signal`, and
+`tokio::sync`. In addition, it adds new unstable APIs to `tokio::task` (`Id`s
+for uniquely identifying a task, and `AbortHandle` for remotely cancelling a
+task), as well as a number of bugfixes.
+
+### Fixed
+
+- blocking: add missing `#[track_caller]` for `spawn_blocking` ([#4616])
+- macros: fix `select` macro to process 64 branches ([#4519])
+- net: fix `try_io` methods not calling Mio's `try_io` internally ([#4582])
+- runtime: recover when OS fails to spawn a new thread ([#4485])
+
+### Added
+
+- net: add `UdpSocket::peer_addr` ([#4611])
+- net: add `try_read_buf` method for named pipes ([#4626])
+- signal: add `SignalKind` `Hash`/`Eq` impls and `c_int` conversion ([#4540])
+- signal: add support for signals up to `SIGRTMAX` ([#4555])
+- sync: add `watch::Sender::send_modify` method ([#4310])
+- sync: add `broadcast::Receiver::len` method ([#4542])
+- sync: add `watch::Receiver::same_channel` method ([#4581])
+- sync: implement `Clone` for `RecvError` types ([#4560])
+
+### Changed
+
+- update `mio` to 0.8.1 ([#4582])
+- macros: rename `tokio::select!`'s internal `util` module ([#4543])
+- runtime: use `Vec::with_capacity` when building runtime ([#4553])
+
+### Documented
+
+- improve docs for `tokio_unstable` ([#4524])
+- runtime: include more documentation for thread_pool/worker ([#4511])
+- runtime: update `Handle::current`'s docs to mention `EnterGuard` ([#4567])
+- time: clarify platform specific timer resolution ([#4474])
+- signal: document that `Signal::recv` is cancel-safe ([#4634])
+- sync: `UnboundedReceiver` close docs ([#4548])
+
+### Unstable
+
+The following changes only apply when building with `--cfg tokio_unstable`:
+
+- task: add `task::Id` type ([#4630])
+- task: add `AbortHandle` type for cancelling tasks in a `JoinSet` ([#4530],
+ [#4640])
+- task: fix missing `doc(cfg(...))` attributes for `JoinSet` ([#4531])
+- task: fix broken link in `AbortHandle` RustDoc ([#4545])
+- metrics: add initial IO driver metrics ([#4507])
+
+
+[#4616]: https://github.com/tokio-rs/tokio/pull/4616
+[#4519]: https://github.com/tokio-rs/tokio/pull/4519
+[#4582]: https://github.com/tokio-rs/tokio/pull/4582
+[#4485]: https://github.com/tokio-rs/tokio/pull/4485
+[#4613]: https://github.com/tokio-rs/tokio/pull/4613
+[#4611]: https://github.com/tokio-rs/tokio/pull/4611
+[#4626]: https://github.com/tokio-rs/tokio/pull/4626
+[#4540]: https://github.com/tokio-rs/tokio/pull/4540
+[#4555]: https://github.com/tokio-rs/tokio/pull/4555
+[#4310]: https://github.com/tokio-rs/tokio/pull/4310
+[#4542]: https://github.com/tokio-rs/tokio/pull/4542
+[#4581]: https://github.com/tokio-rs/tokio/pull/4581
+[#4560]: https://github.com/tokio-rs/tokio/pull/4560
+[#4631]: https://github.com/tokio-rs/tokio/pull/4631
+[#4582]: https://github.com/tokio-rs/tokio/pull/4582
+[#4543]: https://github.com/tokio-rs/tokio/pull/4543
+[#4553]: https://github.com/tokio-rs/tokio/pull/4553
+[#4524]: https://github.com/tokio-rs/tokio/pull/4524
+[#4511]: https://github.com/tokio-rs/tokio/pull/4511
+[#4567]: https://github.com/tokio-rs/tokio/pull/4567
+[#4474]: https://github.com/tokio-rs/tokio/pull/4474
+[#4634]: https://github.com/tokio-rs/tokio/pull/4634
+[#4548]: https://github.com/tokio-rs/tokio/pull/4548
+[#4630]: https://github.com/tokio-rs/tokio/pull/4630
+[#4530]: https://github.com/tokio-rs/tokio/pull/4530
+[#4640]: https://github.com/tokio-rs/tokio/pull/4640
+[#4531]: https://github.com/tokio-rs/tokio/pull/4531
+[#4545]: https://github.com/tokio-rs/tokio/pull/4545
+[#4507]: https://github.com/tokio-rs/tokio/pull/4507
+
+# 1.17.0 (February 16, 2022)
+
+This release updates the minimum supported Rust version (MSRV) to 1.49, the
+`mio` dependency to v0.8, and the (optional) `parking_lot` dependency to v0.12.
+Additionally, it contains several bug fixes, as well as internal refactoring and
+performance improvements.
+
+### Fixed
+
+- time: prevent panicking in `sleep` with large durations ([#4495])
+- time: eliminate potential panics in `Instant` arithmetic on platforms where
+ `Instant::now` is not monotonic ([#4461])
+- io: fix `DuplexStream` not participating in cooperative yielding ([#4478])
+- rt: fix potential double panic when dropping a `JoinHandle` ([#4430])
+
+### Changed
+
+- update minimum supported Rust version to 1.49 ([#4457])
+- update `parking_lot` dependency to v0.12.0 ([#4459])
+- update `mio` dependency to v0.8 ([#4449])
+- rt: remove an unnecessary lock in the blocking pool ([#4436])
+- rt: remove an unnecessary enum in the basic scheduler ([#4462])
+- time: use bit manipulation instead of modulo to improve performance ([#4480])
+- net: use `std::future::Ready` instead of our own `Ready` future ([#4271])
+- replace deprecated `atomic::spin_loop_hint` with `hint::spin_loop` ([#4491])
+- fix miri failures in intrusive linked lists ([#4397])
+
+### Documented
+
+- io: add an example for `tokio::process::ChildStdin` ([#4479])
+
+### Unstable
+
+The following changes only apply when building with `--cfg tokio_unstable`:
+
+- task: fix missing location information in `tracing` spans generated by
+ `spawn_local` ([#4483])
+- task: add `JoinSet` for managing sets of tasks ([#4335])
+- metrics: fix compilation error on MIPS ([#4475])
+- metrics: fix compilation error on arm32v7 ([#4453])
+
+[#4495]: https://github.com/tokio-rs/tokio/pull/4495
+[#4461]: https://github.com/tokio-rs/tokio/pull/4461
+[#4478]: https://github.com/tokio-rs/tokio/pull/4478
+[#4430]: https://github.com/tokio-rs/tokio/pull/4430
+[#4457]: https://github.com/tokio-rs/tokio/pull/4457
+[#4459]: https://github.com/tokio-rs/tokio/pull/4459
+[#4449]: https://github.com/tokio-rs/tokio/pull/4449
+[#4462]: https://github.com/tokio-rs/tokio/pull/4462
+[#4436]: https://github.com/tokio-rs/tokio/pull/4436
+[#4480]: https://github.com/tokio-rs/tokio/pull/4480
+[#4271]: https://github.com/tokio-rs/tokio/pull/4271
+[#4491]: https://github.com/tokio-rs/tokio/pull/4491
+[#4397]: https://github.com/tokio-rs/tokio/pull/4397
+[#4479]: https://github.com/tokio-rs/tokio/pull/4479
+[#4483]: https://github.com/tokio-rs/tokio/pull/4483
+[#4335]: https://github.com/tokio-rs/tokio/pull/4335
+[#4475]: https://github.com/tokio-rs/tokio/pull/4475
+[#4453]: https://github.com/tokio-rs/tokio/pull/4453
+
+# 1.16.1 (January 28, 2022)
+
+This release fixes a bug in [#4428] with the change [#4437].
+
+[#4428]: https://github.com/tokio-rs/tokio/pull/4428
+[#4437]: https://github.com/tokio-rs/tokio/pull/4437
+
+# 1.16.0 (January 27, 2022)
+
+Fixes a soundness bug in `io::Take` ([#4428]). The unsoundness is exposed when
+leaking memory in the given `AsyncRead` implementation and then overwriting the
+supplied buffer:
+
+```rust
+impl AsyncRead for Buggy {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>
+ ) -> Poll<Result<()>> {
+ let new_buf = vec![0; 5].leak();
+ *buf = ReadBuf::new(new_buf);
+ buf.put_slice(b"hello");
+ Poll::Ready(Ok(()))
+ }
+}
+```
+
+Also, this release includes improvements to the multi-threaded scheduler that
+can increase throughput by up to 20% in some cases ([#4383]).
+
+### Fixed
+
+- io: **soundness** don't expose uninitialized memory when using `io::Take` in edge case ([#4428])
+- fs: ensure `File::write` results in a `write` syscall when the runtime shuts down ([#4316])
+- process: drop pipe after child exits in `wait_with_output` ([#4315])
+- rt: improve error message when spawning a thread fails ([#4398])
+- rt: reduce false-positive thread wakups in the multi-threaded scheduler ([#4383])
+- sync: don't inherit `Send` from `parking_lot::*Guard` ([#4359])
+
+### Added
+
+- net: `TcpSocket::linger()` and `set_linger()` ([#4324])
+- net: impl `UnwindSafe` for socket types ([#4384])
+- rt: impl `UnwindSafe` for `JoinHandle` ([#4418])
+- sync: `watch::Receiver::has_changed()` ([#4342])
+- sync: `oneshot::Receiver::blocking_recv()` ([#4334])
+- sync: `RwLock` blocking operations ([#4425])
+
+### Unstable
+
+The following changes only apply when building with `--cfg tokio_unstable`
+
+- rt: **breaking change** overhaul runtime metrics API ([#4373])
+
+[#4428]: https://github.com/tokio-rs/tokio/pull/4428
+[#4316]: https://github.com/tokio-rs/tokio/pull/4316
+[#4315]: https://github.com/tokio-rs/tokio/pull/4315
+[#4398]: https://github.com/tokio-rs/tokio/pull/4398
+[#4383]: https://github.com/tokio-rs/tokio/pull/4383
+[#4359]: https://github.com/tokio-rs/tokio/pull/4359
+[#4324]: https://github.com/tokio-rs/tokio/pull/4324
+[#4384]: https://github.com/tokio-rs/tokio/pull/4384
+[#4418]: https://github.com/tokio-rs/tokio/pull/4418
+[#4342]: https://github.com/tokio-rs/tokio/pull/4342
+[#4334]: https://github.com/tokio-rs/tokio/pull/4334
+[#4425]: https://github.com/tokio-rs/tokio/pull/4425
+[#4373]: https://github.com/tokio-rs/tokio/pull/4373
+
+# 1.15.0 (December 15, 2021)
+
+### Fixed
+
+- io: add cooperative yielding support to `io::empty()` ([#4300])
+- time: make timeout robust against budget-depleting tasks ([#4314])
+
+### Changed
+
+- update minimum supported Rust version to 1.46.
+
+### Added
+
+- time: add `Interval::reset()` ([#4248])
+- io: add explicit lifetimes to `AsyncFdReadyGuard` ([#4267])
+- process: add `Command::as_std()` ([#4295])
+
+### Added (unstable)
+
+- tracing: instrument `tokio::sync` types ([#4302])
+
+[#4302]: https://github.com/tokio-rs/tokio/pull/4302
+[#4300]: https://github.com/tokio-rs/tokio/pull/4300
+[#4295]: https://github.com/tokio-rs/tokio/pull/4295
+[#4267]: https://github.com/tokio-rs/tokio/pull/4267
+[#4248]: https://github.com/tokio-rs/tokio/pull/4248
+[#4314]: https://github.com/tokio-rs/tokio/pull/4314
+
# 1.14.0 (November 15, 2021)
### Fixed
@@ -919,7 +1677,7 @@
- Feature flags are simplified
- `rt-core` and `rt-util` are combined to `rt`
- `rt-threaded` is renamed to `rt-multi-thread` to match builder API
- - `tcp`, `udp`, `uds`, `dns` are combied to `net`.
+ - `tcp`, `udp`, `uds`, `dns` are combined to `net`.
- `parking_lot` is included with `full`
### Changes
@@ -1417,7 +2175,7 @@
- `net::lookup_host` maps a `T: ToSocketAddrs` to a stream of `SocketAddrs` ([#1870]).
- `process::Child` fields are made public to match `std` ([#2014]).
- impl `Stream` for `sync::broadcast::Receiver` ([#2012]).
-- `sync::RwLock` provides an asynchonous read-write lock ([#1699]).
+- `sync::RwLock` provides an asynchronous read-write lock ([#1699]).
- `runtime::Handle::current` returns the handle for the current runtime ([#2040]).
- `StreamExt::filter` filters stream values according to a predicate ([#2001]).
- `StreamExt::filter_map` simultaneously filter and map stream values ([#2001]).
@@ -1526,7 +2284,7 @@
### Fixes
- calling `spawn_blocking` after runtime shutdown ([#1875]).
-- `LocalSet` drop inifinite loop ([#1892]).
+- `LocalSet` drop infinite loop ([#1892]).
- `LocalSet` hang under load ([#1905]).
- improved documentation ([#1865], [#1866], [#1868], [#1874], [#1876], [#1911]).
diff --git a/Cargo.toml b/Cargo.toml
index ec9b335..2ea9473 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,23 +11,48 @@
[package]
edition = "2018"
+rust-version = "1.49"
name = "tokio"
-version = "1.14.0"
+version = "1.25.0"
authors = ["Tokio Contributors <team@tokio.rs>"]
-description = "An event-driven, non-blocking I/O platform for writing asynchronous I/O\nbacked applications.\n"
+description = """
+An event-driven, non-blocking I/O platform for writing asynchronous I/O
+backed applications.
+"""
homepage = "https://tokio.rs"
-documentation = "https://docs.rs/tokio/1.14.0/tokio/"
readme = "README.md"
-keywords = ["io", "async", "non-blocking", "futures"]
-categories = ["asynchronous", "network-programming"]
+keywords = [
+ "io",
+ "async",
+ "non-blocking",
+ "futures",
+]
+categories = [
+ "asynchronous",
+ "network-programming",
+]
license = "MIT"
repository = "https://github.com/tokio-rs/tokio"
+
[package.metadata.docs.rs]
all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
+rustdoc-args = [
+ "--cfg",
+ "docsrs",
+ "--cfg",
+ "tokio_unstable",
+]
+rustc-args = [
+ "--cfg",
+ "tokio_unstable",
+]
[package.metadata.playground]
-features = ["full", "test-util"]
+features = [
+ "full",
+ "test-util",
+]
+
[dependencies.bytes]
version = "1.0.0"
optional = true
@@ -37,27 +62,24 @@
optional = true
[dependencies.mio]
-version = "0.7.6"
+version = "0.8.4"
optional = true
[dependencies.num_cpus]
version = "1.8.0"
optional = true
-[dependencies.once_cell]
-version = "1.5.2"
-optional = true
-
[dependencies.parking_lot]
-version = "0.11.0"
+version = "0.12.0"
optional = true
[dependencies.pin-project-lite]
version = "0.2.0"
[dependencies.tokio-macros]
-version = "1.6.0"
+version = "1.7.0"
optional = true
+
[dev-dependencies.async-stream]
version = "0.3"
@@ -66,16 +88,7 @@
features = ["async-await"]
[dev-dependencies.mockall]
-version = "0.10.2"
-
-[dev-dependencies.proptest]
-version = "1"
-
-[dev-dependencies.rand]
-version = "0.8.0"
-
-[dev-dependencies.socket2]
-version = "0.4"
+version = "0.11.1"
[dev-dependencies.tempfile]
version = "3.1.0"
@@ -85,36 +98,120 @@
[dev-dependencies.tokio-test]
version = "0.4.0"
+
[build-dependencies.autocfg]
-version = "1"
+version = "1.1"
[features]
default = []
fs = []
-full = ["fs", "io-util", "io-std", "macros", "net", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "sync", "time"]
+full = [
+ "fs",
+ "io-util",
+ "io-std",
+ "macros",
+ "net",
+ "parking_lot",
+ "process",
+ "rt",
+ "rt-multi-thread",
+ "signal",
+ "sync",
+ "time",
+]
io-std = []
-io-util = ["memchr", "bytes"]
+io-util = [
+ "memchr",
+ "bytes",
+]
macros = ["tokio-macros"]
-net = ["libc", "mio/os-poll", "mio/os-util", "mio/tcp", "mio/udp", "mio/uds", "winapi/namedpipeapi"]
-process = ["bytes", "once_cell", "libc", "mio/os-poll", "mio/os-util", "mio/uds", "signal-hook-registry", "winapi/threadpoollegacyapiset"]
+net = [
+ "libc",
+ "mio/os-poll",
+ "mio/os-ext",
+ "mio/net",
+ "socket2",
+ "windows-sys/Win32_Foundation",
+ "windows-sys/Win32_Security",
+ "windows-sys/Win32_Storage_FileSystem",
+ "windows-sys/Win32_System_Pipes",
+ "windows-sys/Win32_System_SystemServices",
+]
+process = [
+ "bytes",
+ "libc",
+ "mio/os-poll",
+ "mio/os-ext",
+ "mio/net",
+ "signal-hook-registry",
+ "windows-sys/Win32_Foundation",
+ "windows-sys/Win32_System_Threading",
+ "windows-sys/Win32_System_WindowsProgramming",
+]
rt = []
-rt-multi-thread = ["num_cpus", "rt"]
-signal = ["once_cell", "libc", "mio/os-poll", "mio/uds", "mio/os-util", "signal-hook-registry", "winapi/consoleapi"]
+rt-multi-thread = [
+ "num_cpus",
+ "rt",
+]
+signal = [
+ "libc",
+ "mio/os-poll",
+ "mio/net",
+ "mio/os-ext",
+ "signal-hook-registry",
+ "windows-sys/Win32_Foundation",
+ "windows-sys/Win32_System_Console",
+]
stats = []
sync = []
-test-util = ["rt", "sync", "time"]
+test-util = [
+ "rt",
+ "sync",
+ "time",
+]
time = []
+
+[target."cfg(all(any(target_arch = \"wasm32\", target_arch = \"wasm64\"), not(target_os = \"wasi\")))".dev-dependencies.wasm-bindgen-test]
+version = "0.3.0"
+
+[target."cfg(docsrs)".dependencies.windows-sys]
+version = "0.42.0"
+features = [
+ "Win32_Foundation",
+ "Win32_Security_Authorization",
+]
+
[target."cfg(loom)".dev-dependencies.loom]
-version = "0.5"
-features = ["futures", "checkpoint"]
+version = "0.5.2"
+features = [
+ "futures",
+ "checkpoint",
+]
+
+[target."cfg(not(all(any(target_arch = \"wasm32\", target_arch = \"wasm64\"), target_os = \"unknown\")))".dev-dependencies.rand]
+version = "0.8.0"
+
+[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dependencies.socket2]
+version = "0.4.4"
+features = ["all"]
+optional = true
+
+[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dev-dependencies.proptest]
+version = "1"
+
+[target."cfg(not(any(target_arch = \"wasm32\", target_arch = \"wasm64\")))".dev-dependencies.socket2]
+version = "0.4"
+
[target."cfg(target_os = \"freebsd\")".dev-dependencies.mio-aio]
-version = "0.6.0"
+version = "0.7.0"
features = ["tokio"]
+
[target."cfg(tokio_unstable)".dependencies.tracing]
version = "0.1.25"
features = ["std"]
optional = true
default-features = false
+
[target."cfg(unix)".dependencies.libc]
version = "0.2.42"
optional = true
@@ -122,14 +219,21 @@
[target."cfg(unix)".dependencies.signal-hook-registry]
version = "1.1.1"
optional = true
+
[target."cfg(unix)".dev-dependencies.libc]
version = "0.2.42"
[target."cfg(unix)".dev-dependencies.nix]
-version = "0.22.0"
-[target."cfg(windows)".dependencies.winapi]
-version = "0.3.8"
-optional = true
+version = "0.26"
+features = [
+ "fs",
+ "socket",
+]
default-features = false
+
+[target."cfg(windows)".dependencies.windows-sys]
+version = "0.42.0"
+optional = true
+
[target."cfg(windows)".dev-dependencies.ntapi]
version = "0.3.6"
diff --git a/Cargo.toml.orig b/Cargo.toml.orig
index 348ec46..0f6d30a 100644
--- a/Cargo.toml.orig
+++ b/Cargo.toml.orig
@@ -3,16 +3,15 @@
# When releasing to crates.io:
# - Remove path dependencies
# - Update doc url
-# - Cargo.toml
# - README.md
# - Update CHANGELOG.md.
-# - Create "v1.0.x" git tag.
-version = "1.14.0"
+# - Create "v1.x.y" git tag.
+version = "1.25.0"
edition = "2018"
+rust-version = "1.49"
authors = ["Tokio Contributors <team@tokio.rs>"]
license = "MIT"
readme = "README.md"
-documentation = "https://docs.rs/tokio/1.14.0/tokio/"
repository = "https://github.com/tokio-rs/tokio"
homepage = "https://tokio.rs"
description = """
@@ -47,25 +46,28 @@
# stdin, stdout, stderr
io-std = []
macros = ["tokio-macros"]
-stats = []
net = [
"libc",
"mio/os-poll",
- "mio/os-util",
- "mio/tcp",
- "mio/udp",
- "mio/uds",
- "winapi/namedpipeapi",
+ "mio/os-ext",
+ "mio/net",
+ "socket2",
+ "windows-sys/Win32_Foundation",
+ "windows-sys/Win32_Security",
+ "windows-sys/Win32_Storage_FileSystem",
+ "windows-sys/Win32_System_Pipes",
+ "windows-sys/Win32_System_SystemServices",
]
process = [
"bytes",
- "once_cell",
"libc",
"mio/os-poll",
- "mio/os-util",
- "mio/uds",
+ "mio/os-ext",
+ "mio/net",
"signal-hook-registry",
- "winapi/threadpoollegacyapiset",
+ "windows-sys/Win32_Foundation",
+ "windows-sys/Win32_System_Threading",
+ "windows-sys/Win32_System_WindowsProgramming",
]
# Includes basic task execution capabilities
rt = []
@@ -74,30 +76,40 @@
"rt",
]
signal = [
- "once_cell",
"libc",
"mio/os-poll",
- "mio/uds",
- "mio/os-util",
+ "mio/net",
+ "mio/os-ext",
"signal-hook-registry",
- "winapi/consoleapi",
+ "windows-sys/Win32_Foundation",
+ "windows-sys/Win32_System_Console",
]
sync = []
test-util = ["rt", "sync", "time"]
time = []
+# Technically, removing this is a breaking change even though it only ever did
+# anything with the unstable flag on. It is probably safe to get rid of it after
+# a few releases.
+stats = []
+
+[build-dependencies]
+autocfg = "1.1"
+
[dependencies]
-tokio-macros = { version = "1.6.0", path = "../tokio-macros", optional = true }
+tokio-macros = { version = "1.7.0", path = "../tokio-macros", optional = true }
pin-project-lite = "0.2.0"
# Everything else is optional...
bytes = { version = "1.0.0", optional = true }
-once_cell = { version = "1.5.2", optional = true }
memchr = { version = "2.2", optional = true }
-mio = { version = "0.7.6", optional = true }
+mio = { version = "0.8.4", optional = true }
num_cpus = { version = "1.8.0", optional = true }
-parking_lot = { version = "0.11.0", optional = true }
+parking_lot = { version = "0.12.0", optional = true }
+
+[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dependencies]
+socket2 = { version = "0.4.4", optional = true, features = [ "all" ] }
# Currently unstable. The API exposed by these features may be broken at any time.
# Requires `--cfg tokio_unstable` to enable.
@@ -110,13 +122,19 @@
[target.'cfg(unix)'.dev-dependencies]
libc = { version = "0.2.42" }
-nix = { version = "0.22.0" }
+nix = { version = "0.26", default-features = false, features = ["fs", "socket"] }
-[target.'cfg(windows)'.dependencies.winapi]
-version = "0.3.8"
-default-features = false
+[target.'cfg(windows)'.dependencies.windows-sys]
+version = "0.42.0"
optional = true
+[target.'cfg(docsrs)'.dependencies.windows-sys]
+version = "0.42.0"
+features = [
+ "Win32_Foundation",
+ "Win32_Security_Authorization",
+]
+
[target.'cfg(windows)'.dev-dependencies.ntapi]
version = "0.3.6"
@@ -124,25 +142,33 @@
tokio-test = { version = "0.4.0", path = "../tokio-test" }
tokio-stream = { version = "0.1", path = "../tokio-stream" }
futures = { version = "0.3.0", features = ["async-await"] }
-mockall = "0.10.2"
-proptest = "1"
-rand = "0.8.0"
+mockall = "0.11.1"
tempfile = "3.1.0"
async-stream = "0.3"
+
+[target.'cfg(not(any(target_arch = "wasm32", target_arch = "wasm64")))'.dev-dependencies]
+proptest = "1"
socket2 = "0.4"
+[target.'cfg(not(all(any(target_arch = "wasm32", target_arch = "wasm64"), target_os = "unknown")))'.dev-dependencies]
+rand = "0.8.0"
+
+[target.'cfg(all(any(target_arch = "wasm32", target_arch = "wasm64"), not(target_os = "wasi")))'.dev-dependencies]
+wasm-bindgen-test = "0.3.0"
+
[target.'cfg(target_os = "freebsd")'.dev-dependencies]
-mio-aio = { version = "0.6.0", features = ["tokio"] }
+mio-aio = { version = "0.7.0", features = ["tokio"] }
[target.'cfg(loom)'.dev-dependencies]
-loom = { version = "0.5", features = ["futures", "checkpoint"] }
-
-[build-dependencies]
-autocfg = "1" # Needed for conditionally enabling `track-caller`
+loom = { version = "0.5.2", features = ["futures", "checkpoint"] }
[package.metadata.docs.rs]
all-features = true
-rustdoc-args = ["--cfg", "docsrs"]
+# enable unstable features in the documentation
+rustdoc-args = ["--cfg", "docsrs", "--cfg", "tokio_unstable"]
+# it's necessary to _also_ pass `--cfg tokio_unstable` to rustc, or else
+# dependencies will not be enabled, and the docs build will fail.
+rustc-args = ["--cfg", "tokio_unstable"]
[package.metadata.playground]
features = ["full", "test-util"]
diff --git a/LICENSE b/LICENSE
index ffa38bb..8bdf6bd 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2021 Tokio Contributors
+Copyright (c) 2023 Tokio Contributors
Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated
diff --git a/METADATA b/METADATA
index 1d167d1..b8a5d8a 100644
--- a/METADATA
+++ b/METADATA
@@ -1,3 +1,7 @@
+# This project was upgraded with external_updater.
+# Usage: tools/external_updater/updater.sh update rust/crates/tokio
+# For more info, check https://cs.android.com/android/platform/superproject/+/master:tools/external_updater/README.md
+
name: "tokio"
description: "An event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications."
third_party {
@@ -7,13 +11,13 @@
}
url {
type: ARCHIVE
- value: "https://static.crates.io/crates/tokio/tokio-1.14.0.crate"
+ value: "https://static.crates.io/crates/tokio/tokio-1.25.0.crate"
}
- version: "1.14.0"
+ version: "1.25.0"
license_type: NOTICE
last_upgrade_date {
- year: 2021
- month: 11
- day: 16
+ year: 2023
+ month: 2
+ day: 6
}
}
diff --git a/README.md b/README.md
index 19f049c..462e6e8 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,7 @@
```toml
[dependencies]
-tokio = { version = "1.14.0", features = ["full"] }
+tokio = { version = "1.25.0", features = ["full"] }
```
Then, on your main.rs:
@@ -161,11 +161,33 @@
[`mio`]: https://github.com/tokio-rs/mio
[`bytes`]: https://github.com/tokio-rs/bytes
+## Changelog
+
+The Tokio repository contains multiple crates. Each crate has its own changelog.
+
+ * `tokio` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio/CHANGELOG.md)
+ * `tokio-util` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-util/CHANGELOG.md)
+ * `tokio-stream` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-stream/CHANGELOG.md)
+ * `tokio-macros` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-macros/CHANGELOG.md)
+ * `tokio-test` - [view changelog](https://github.com/tokio-rs/tokio/blob/master/tokio-test/CHANGELOG.md)
+
## Supported Rust Versions
-Tokio is built against the latest stable release. The minimum supported version
-is 1.45. The current Tokio version is not guaranteed to build on Rust versions
-earlier than the minimum supported version.
+<!--
+When updating this, also update:
+- .github/workflows/ci.yml
+- CONTRIBUTING.md
+- README.md
+- tokio/README.md
+- tokio/Cargo.toml
+- tokio-util/Cargo.toml
+- tokio-test/Cargo.toml
+- tokio-stream/Cargo.toml
+-->
+
+Tokio will keep a rolling MSRV (minimum supported rust version) policy of **at
+least** 6 months. When increasing the MSRV, the new Rust version must have been
+released at least six months ago. The current MSRV is 1.49.0.
## Release schedule
@@ -180,17 +202,18 @@
released as a new patch release for each LTS minor version. Our current LTS
releases are:
- * `1.8.x` - LTS release until February 2022.
+ * `1.18.x` - LTS release until June 2023
+ * `1.20.x` - LTS release until September 2023.
-Each LTS release will continue to receive backported fixes for at least half a
-year. If you wish to use a fixed minor release in your project, we recommend
-that you use an LTS release.
+Each LTS release will continue to receive backported fixes for at least a year.
+If you wish to use a fixed minor release in your project, we recommend that you
+use an LTS release.
To use a fixed minor version, you can specify the version with a tilde. For
-example, to specify that you wish to use the newest `1.8.x` patch release, you
+example, to specify that you wish to use the newest `1.18.x` patch release, you
can use the following dependency specification:
```text
-tokio = { version = "~1.8", features = [...] }
+tokio = { version = "~1.18", features = [...] }
```
## License
diff --git a/TEST_MAPPING b/TEST_MAPPING
index 2e7046e..63733b6 100644
--- a/TEST_MAPPING
+++ b/TEST_MAPPING
@@ -2,20 +2,23 @@
{
"imports": [
{
+ "path": "external/rust/crates/async-stream"
+ },
+ {
"path": "external/rust/crates/futures-util"
},
{
"path": "external/rust/crates/tokio-test"
+ },
+ {
+ "path": "external/uwb/src"
+ },
+ {
+ "path": "packages/modules/DnsResolver"
}
],
"presubmit": [
{
- "name": "doh_unit_test"
- },
- {
- "name": "rustBinderTest"
- },
- {
"name": "tokio_test_tests__require_full"
},
{
@@ -133,9 +136,6 @@
"name": "tokio_test_tests_sync_errors"
},
{
- "name": "tokio_test_tests_sync_mpsc"
- },
- {
"name": "tokio_test_tests_sync_mutex"
},
{
@@ -213,12 +213,6 @@
],
"presubmit-rust": [
{
- "name": "doh_unit_test"
- },
- {
- "name": "rustBinderTest"
- },
- {
"name": "tokio_test_tests__require_full"
},
{
@@ -336,9 +330,6 @@
"name": "tokio_test_tests_sync_errors"
},
{
- "name": "tokio_test_tests_sync_mpsc"
- },
- {
"name": "tokio_test_tests_sync_mutex"
},
{
diff --git a/build.rs b/build.rs
index fe5c830..ddade28 100644
--- a/build.rs
+++ b/build.rs
@@ -1,11 +1,121 @@
use autocfg::AutoCfg;
+const CONST_THREAD_LOCAL_PROBE: &str = r#"
+{
+ thread_local! {
+ static MY_PROBE: usize = const { 10 };
+ }
+
+ MY_PROBE.with(|val| *val)
+}
+"#;
+
+const ADDR_OF_PROBE: &str = r#"
+{
+ let my_var = 10;
+ ::std::ptr::addr_of!(my_var)
+}
+"#;
+
+const CONST_MUTEX_NEW_PROBE: &str = r#"
+{
+ static MY_MUTEX: ::std::sync::Mutex<i32> = ::std::sync::Mutex::new(1);
+ *MY_MUTEX.lock().unwrap()
+}
+"#;
+
+const TARGET_HAS_ATOMIC_PROBE: &str = r#"
+{
+ #[cfg(target_has_atomic = "ptr")]
+ let _ = ();
+}
+"#;
+
+const TARGET_ATOMIC_U64_PROBE: &str = r#"
+{
+ #[allow(unused_imports)]
+ use std::sync::atomic::AtomicU64 as _;
+}
+"#;
+
fn main() {
+ let mut enable_const_thread_local = false;
+ let mut enable_addr_of = false;
+ let mut enable_target_has_atomic = false;
+ let mut enable_const_mutex_new = false;
+ let mut target_needs_atomic_u64_fallback = false;
+
match AutoCfg::new() {
Ok(ac) => {
- // The #[track_caller] attribute was stabilized in rustc 1.46.0.
- if ac.probe_rustc_version(1, 46) {
- autocfg::emit("tokio_track_caller")
+ // These checks prefer to call only `probe_rustc_version` if that is
+ // enough to determine whether the feature is supported. This is
+ // because the `probe_expression` call involves a call to rustc,
+ // which the `probe_rustc_version` call avoids.
+
+ // Const-initialized thread locals were stabilized in 1.59.
+ if ac.probe_rustc_version(1, 60) {
+ enable_const_thread_local = true;
+ } else if ac.probe_rustc_version(1, 59) {
+ // This compiler claims to be 1.59, but there are some nightly
+ // compilers that claim to be 1.59 without supporting the
+ // feature. Explicitly probe to check if code using them
+ // compiles.
+ //
+ // The oldest nightly that supports the feature is 2021-12-06.
+ if ac.probe_expression(CONST_THREAD_LOCAL_PROBE) {
+ enable_const_thread_local = true;
+ }
+ }
+
+ // The `addr_of` and `addr_of_mut` macros were stabilized in 1.51.
+ if ac.probe_rustc_version(1, 52) {
+ enable_addr_of = true;
+ } else if ac.probe_rustc_version(1, 51) {
+ // This compiler claims to be 1.51, but there are some nightly
+ // compilers that claim to be 1.51 without supporting the
+ // feature. Explicitly probe to check if code using them
+ // compiles.
+ //
+ // The oldest nightly that supports the feature is 2021-01-31.
+ if ac.probe_expression(ADDR_OF_PROBE) {
+ enable_addr_of = true;
+ }
+ }
+
+ // The `target_has_atomic` cfg was stabilized in 1.60.
+ if ac.probe_rustc_version(1, 61) {
+ enable_target_has_atomic = true;
+ } else if ac.probe_rustc_version(1, 60) {
+ // This compiler claims to be 1.60, but there are some nightly
+ // compilers that claim to be 1.60 without supporting the
+ // feature. Explicitly probe to check if code using them
+ // compiles.
+ //
+ // The oldest nightly that supports the feature is 2022-02-11.
+ if ac.probe_expression(TARGET_HAS_ATOMIC_PROBE) {
+ enable_target_has_atomic = true;
+ }
+ }
+
+ // If we can't tell using `target_has_atomic`, tell if the target
+ // has `AtomicU64` by trying to use it.
+ if !enable_target_has_atomic && !ac.probe_expression(TARGET_ATOMIC_U64_PROBE) {
+ target_needs_atomic_u64_fallback = true;
+ }
+
+ // The `Mutex::new` method was made const in 1.63.
+ if ac.probe_rustc_version(1, 64) {
+ enable_const_mutex_new = true;
+ } else if ac.probe_rustc_version(1, 63) {
+ // This compiler claims to be 1.63, but there are some nightly
+ // compilers that claim to be 1.63 without supporting the
+ // feature. Explicitly probe to check if code using them
+ // compiles.
+ //
+ // The oldest nightly that supports the feature is 2022-06-20.
+ if ac.probe_expression(CONST_MUTEX_NEW_PROBE) {
+ enable_const_mutex_new = true;
+ }
}
}
@@ -19,4 +129,57 @@
);
}
}
+
+ if !enable_const_thread_local {
+ // To disable this feature on compilers that support it, you can
+ // explicitly pass this flag with the following environment variable:
+ //
+ // RUSTFLAGS="--cfg tokio_no_const_thread_local"
+ autocfg::emit("tokio_no_const_thread_local")
+ }
+
+ if !enable_addr_of {
+ // To disable this feature on compilers that support it, you can
+ // explicitly pass this flag with the following environment variable:
+ //
+ // RUSTFLAGS="--cfg tokio_no_addr_of"
+ autocfg::emit("tokio_no_addr_of")
+ }
+
+ if !enable_target_has_atomic {
+ // To disable this feature on compilers that support it, you can
+ // explicitly pass this flag with the following environment variable:
+ //
+ // RUSTFLAGS="--cfg tokio_no_target_has_atomic"
+ autocfg::emit("tokio_no_target_has_atomic")
+ }
+
+ if !enable_const_mutex_new {
+ // To disable this feature on compilers that support it, you can
+ // explicitly pass this flag with the following environment variable:
+ //
+ // RUSTFLAGS="--cfg tokio_no_const_mutex_new"
+ autocfg::emit("tokio_no_const_mutex_new")
+ }
+
+ if target_needs_atomic_u64_fallback {
+ // To disable this feature on compilers that support it, you can
+ // explicitly pass this flag with the following environment variable:
+ //
+ // RUSTFLAGS="--cfg tokio_no_atomic_u64"
+ autocfg::emit("tokio_no_atomic_u64")
+ }
+
+ let target = ::std::env::var("TARGET").unwrap_or_default();
+
+ // We emit cfgs instead of using `target_family = "wasm"` that requires Rust 1.54.
+ // Note that these cfgs are unavailable in `Cargo.toml`.
+ if target.starts_with("wasm") {
+ autocfg::emit("tokio_wasm");
+ if target.contains("wasi") {
+ autocfg::emit("tokio_wasi");
+ } else {
+ autocfg::emit("tokio_wasm_not_wasi");
+ }
+ }
}
diff --git a/cargo2android.json b/cargo2android.json
index 75772f0..443e022 100644
--- a/cargo2android.json
+++ b/cargo2android.json
@@ -2,12 +2,12 @@
"add-toplevel-block": "cargo2android_tests.bp",
"apex-available": [
"//apex_available:platform",
- "com.android.bluetooth",
+ "com.android.btservices",
"com.android.resolv",
"com.android.uwb"
],
"device": true,
- "features": "io-util,macros,rt-multi-thread,sync,net,fs,time",
+ "features": "fs,io-util,macros,net,rt,rt-multi-thread,sync,time",
"min-sdk-version": "29",
"vendor-available": true,
"run": true
diff --git a/cargo2android_tests.bp b/cargo2android_tests.bp
index 809c899..44bc4c4 100644
--- a/cargo2android_tests.bp
+++ b/cargo2android_tests.bp
@@ -1,5 +1,5 @@
rust_defaults {
- name: "tokio_defaults_tokio",
+ name: "tokio_defaults_tests",
crate_name: "tokio",
cargo_env_compat: true,
test_suites: ["general-tests"],
@@ -43,7 +43,7 @@
rust_test {
name: "tokio_test_tests__require_full",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/_require_full.rs"],
test_options: {
@@ -53,7 +53,7 @@
rust_test {
name: "tokio_test_tests_buffered",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/buffered.rs"],
test_options: {
@@ -63,7 +63,7 @@
rust_test {
name: "tokio_test_tests_io_async_fd",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_async_fd.rs"],
test_options: {
@@ -73,7 +73,7 @@
rust_test {
name: "tokio_test_tests_io_async_read",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_async_read.rs"],
test_options: {
@@ -83,7 +83,7 @@
rust_test {
name: "tokio_test_tests_io_chain",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_chain.rs"],
test_options: {
@@ -93,7 +93,7 @@
rust_test {
name: "tokio_test_tests_io_copy",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_copy.rs"],
test_options: {
@@ -103,7 +103,7 @@
rust_test {
name: "tokio_test_tests_io_copy_bidirectional",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_copy_bidirectional.rs"],
test_options: {
@@ -113,7 +113,7 @@
rust_test {
name: "tokio_test_tests_io_driver",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_driver.rs"],
test_options: {
@@ -123,7 +123,7 @@
rust_test {
name: "tokio_test_tests_io_driver_drop",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_driver_drop.rs"],
test_options: {
@@ -133,7 +133,7 @@
rust_test {
name: "tokio_test_tests_io_lines",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_lines.rs"],
test_options: {
@@ -143,7 +143,7 @@
rust_test {
name: "tokio_test_tests_io_mem_stream",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_mem_stream.rs"],
test_options: {
@@ -153,7 +153,7 @@
rust_test {
name: "tokio_test_tests_io_read",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read.rs"],
test_options: {
@@ -163,7 +163,7 @@
rust_test {
name: "tokio_test_tests_io_read_buf",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_buf.rs"],
test_options: {
@@ -173,7 +173,7 @@
rust_test {
name: "tokio_test_tests_io_read_exact",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_exact.rs"],
test_options: {
@@ -183,7 +183,7 @@
rust_test {
name: "tokio_test_tests_io_read_line",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_line.rs"],
test_options: {
@@ -193,7 +193,7 @@
rust_test {
name: "tokio_test_tests_io_read_to_end",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_to_end.rs"],
test_options: {
@@ -203,7 +203,7 @@
rust_test {
name: "tokio_test_tests_io_read_to_string",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_to_string.rs"],
test_options: {
@@ -213,7 +213,7 @@
rust_test {
name: "tokio_test_tests_io_read_until",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_read_until.rs"],
test_options: {
@@ -223,7 +223,7 @@
rust_test {
name: "tokio_test_tests_io_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_split.rs"],
test_options: {
@@ -233,7 +233,7 @@
rust_test {
name: "tokio_test_tests_io_take",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_take.rs"],
test_options: {
@@ -243,7 +243,7 @@
rust_test {
name: "tokio_test_tests_io_write",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write.rs"],
test_options: {
@@ -253,7 +253,7 @@
rust_test {
name: "tokio_test_tests_io_write_all",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write_all.rs"],
test_options: {
@@ -263,7 +263,7 @@
rust_test {
name: "tokio_test_tests_io_write_buf",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write_buf.rs"],
test_options: {
@@ -273,7 +273,7 @@
rust_test {
name: "tokio_test_tests_io_write_int",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/io_write_int.rs"],
test_options: {
@@ -283,7 +283,7 @@
rust_test {
name: "tokio_test_tests_macros_join",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_join.rs"],
test_options: {
@@ -293,7 +293,7 @@
rust_test {
name: "tokio_test_tests_macros_pin",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_pin.rs"],
test_options: {
@@ -303,7 +303,7 @@
rust_test {
name: "tokio_test_tests_macros_select",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_select.rs"],
test_options: {
@@ -313,7 +313,7 @@
rust_test {
name: "tokio_test_tests_macros_test",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_test.rs"],
test_options: {
@@ -323,7 +323,7 @@
rust_test {
name: "tokio_test_tests_macros_try_join",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/macros_try_join.rs"],
test_options: {
@@ -333,7 +333,7 @@
rust_test {
name: "tokio_test_tests_net_bind_resource",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/net_bind_resource.rs"],
test_options: {
@@ -343,7 +343,7 @@
rust_test {
name: "tokio_test_tests_net_lookup_host",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/net_lookup_host.rs"],
test_options: {
@@ -353,7 +353,7 @@
rust_test {
name: "tokio_test_tests_no_rt",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/no_rt.rs"],
test_options: {
@@ -363,7 +363,7 @@
rust_test {
name: "tokio_test_tests_process_kill_on_drop",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/process_kill_on_drop.rs"],
test_options: {
@@ -373,7 +373,7 @@
rust_test {
name: "tokio_test_tests_rt_basic",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/rt_basic.rs"],
test_options: {
@@ -383,7 +383,7 @@
rust_test {
name: "tokio_test_tests_rt_common",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/rt_common.rs"],
test_options: {
@@ -393,7 +393,7 @@
rust_test {
name: "tokio_test_tests_rt_threaded",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/rt_threaded.rs"],
test_options: {
@@ -403,7 +403,7 @@
rust_test {
name: "tokio_test_tests_sync_barrier",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_barrier.rs"],
test_options: {
@@ -413,7 +413,7 @@
rust_test {
name: "tokio_test_tests_sync_broadcast",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_broadcast.rs"],
test_options: {
@@ -423,7 +423,7 @@
rust_test {
name: "tokio_test_tests_sync_errors",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_errors.rs"],
test_options: {
@@ -432,18 +432,8 @@
}
rust_test {
- name: "tokio_test_tests_sync_mpsc",
- defaults: ["tokio_defaults_tokio"],
- host_supported: true,
- srcs: ["tests/sync_mpsc.rs"],
- test_options: {
- unit_test: true,
- },
-}
-
-rust_test {
name: "tokio_test_tests_sync_mutex",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_mutex.rs"],
test_options: {
@@ -453,7 +443,7 @@
rust_test {
name: "tokio_test_tests_sync_mutex_owned",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_mutex_owned.rs"],
test_options: {
@@ -463,7 +453,7 @@
rust_test {
name: "tokio_test_tests_sync_notify",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_notify.rs"],
test_options: {
@@ -473,7 +463,7 @@
rust_test {
name: "tokio_test_tests_sync_oneshot",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_oneshot.rs"],
test_options: {
@@ -483,7 +473,7 @@
rust_test {
name: "tokio_test_tests_sync_rwlock",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_rwlock.rs"],
test_options: {
@@ -493,7 +483,7 @@
rust_test {
name: "tokio_test_tests_sync_semaphore",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_semaphore.rs"],
test_options: {
@@ -503,7 +493,7 @@
rust_test {
name: "tokio_test_tests_sync_semaphore_owned",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_semaphore_owned.rs"],
test_options: {
@@ -513,7 +503,7 @@
rust_test {
name: "tokio_test_tests_sync_watch",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/sync_watch.rs"],
test_options: {
@@ -523,7 +513,7 @@
rust_test {
name: "tokio_test_tests_task_abort",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_abort.rs"],
test_options: {
@@ -533,7 +523,7 @@
rust_test {
name: "tokio_test_tests_task_blocking",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_blocking.rs"],
test_options: {
@@ -543,7 +533,7 @@
rust_test {
name: "tokio_test_tests_task_local",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_local.rs"],
test_options: {
@@ -553,7 +543,7 @@
rust_test {
name: "tokio_test_tests_task_local_set",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/task_local_set.rs"],
test_options: {
@@ -563,7 +553,7 @@
rust_test {
name: "tokio_test_tests_tcp_accept",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_accept.rs"],
test_options: {
@@ -573,7 +563,7 @@
rust_test {
name: "tokio_test_tests_tcp_connect",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_connect.rs"],
test_options: {
@@ -583,7 +573,7 @@
rust_test {
name: "tokio_test_tests_tcp_echo",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_echo.rs"],
test_options: {
@@ -593,7 +583,7 @@
rust_test {
name: "tokio_test_tests_tcp_into_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_into_split.rs"],
test_options: {
@@ -603,7 +593,7 @@
rust_test {
name: "tokio_test_tests_tcp_into_std",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_into_std.rs"],
test_options: {
@@ -613,7 +603,7 @@
rust_test {
name: "tokio_test_tests_tcp_peek",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_peek.rs"],
test_options: {
@@ -623,7 +613,7 @@
rust_test {
name: "tokio_test_tests_tcp_shutdown",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_shutdown.rs"],
test_options: {
@@ -633,7 +623,7 @@
rust_test {
name: "tokio_test_tests_tcp_socket",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_socket.rs"],
test_options: {
@@ -643,7 +633,7 @@
rust_test {
name: "tokio_test_tests_tcp_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/tcp_split.rs"],
test_options: {
@@ -653,7 +643,7 @@
rust_test {
name: "tokio_test_tests_time_rt",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/time_rt.rs"],
test_options: {
@@ -663,7 +653,7 @@
rust_test {
name: "tokio_test_tests_udp",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/udp.rs"],
test_options: {
@@ -673,7 +663,7 @@
rust_test {
name: "tokio_test_tests_uds_cred",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/uds_cred.rs"],
test_options: {
@@ -683,7 +673,7 @@
rust_test {
name: "tokio_test_tests_uds_split",
- defaults: ["tokio_defaults_tokio"],
+ defaults: ["tokio_defaults_tests"],
host_supported: true,
srcs: ["tests/uds_split.rs"],
test_options: {
diff --git a/docs/reactor-refactor.md b/docs/reactor-refactor.md
index 1c9ace1..77e64f4 100644
--- a/docs/reactor-refactor.md
+++ b/docs/reactor-refactor.md
@@ -188,12 +188,12 @@
The `ScheduledIo` readiness `AtomicUsize` is structured as:
```
-| reserved | generation | driver tick | readinesss |
-|----------+------------+--------------+------------|
-| 1 bit | 7 bits + 8 bits + 16 bits |
+| shutdown | generation | driver tick | readiness |
+|----------+------------+--------------+-----------|
+| 1 bit | 7 bits + 8 bits + 16 bits |
```
-The `reserved` and `generation` components exist today.
+The `shutdown` and `generation` components exist today.
The `readiness()` function returns a `ReadyEvent` value. This value includes the
`tick` component read with the resource's readiness value. When
diff --git a/external-types.toml b/external-types.toml
new file mode 100644
index 0000000..a5bde8e
--- /dev/null
+++ b/external-types.toml
@@ -0,0 +1,11 @@
+# This config file is for the `cargo-check-external-types` tool that is run in CI.
+
+# The following are types that are allowed to be exposed in Tokio's public API.
+# The standard library is allowed by default.
+allowed_external_types = [
+ "bytes::buf::buf_impl::Buf",
+ "bytes::buf::buf_mut::BufMut",
+
+ "tokio_macros::*",
+]
+
diff --git a/patches/Android.bp.patch b/patches/Android.bp.patch
new file mode 100644
index 0000000..68fc808
--- /dev/null
+++ b/patches/Android.bp.patch
@@ -0,0 +1,39 @@
+diff --git a/Android.bp b/Android.bp
+index 7d066e6..58422c6 100644
+--- a/Android.bp
++++ b/Android.bp
+@@ -18,10 +18,9 @@
+ ],
+ }
+
+-rust_library {
+- name: "libtokio",
++rust_defaults {
++ name: "tokio_defaults",
+ host_supported: true,
+- crate_name: "tokio",
+ cargo_env_compat: true,
+ cargo_pkg_version: "1.25.0",
+ srcs: ["src/lib.rs"],
+@@ -64,6 +63,21 @@
+ min_sdk_version: "29",
+ }
+
++rust_library {
++ name: "libtokio",
++ crate_name: "tokio",
++ defaults: ["tokio_defaults"],
++}
++
++rust_library {
++ name: "libtokio_for_test",
++ crate_name: "tokio",
++ defaults: ["tokio_defaults"],
++ features: [
++ "test-util",
++ ],
++}
++
+ rust_defaults {
+ name: "tokio_defaults_tests",
+ crate_name: "tokio",
diff --git a/patches/macros_join.rs.patch b/patches/macros_join.rs.patch
new file mode 100644
index 0000000..f041b14
--- /dev/null
+++ b/patches/macros_join.rs.patch
@@ -0,0 +1,12 @@
+diff --git a/tests/macros_join.rs b/tests/macros_join.rs
+index 16e7c43..4441582 100644
+--- a/tests/macros_join.rs
++++ b/tests/macros_join.rs
+@@ -66,6 +66,7 @@ async fn two_await() {
+
+ #[test]
+ #[cfg(target_pointer_width = "64")]
++#[ignore = "Android: ignore until the compiler is updated. aliceryhl@ says these tests assume latest stable compiler."]
+ fn join_size() {
+ use futures::future;
+ use std::mem;
diff --git a/patches/panic_unwind_tests.patch b/patches/panic_unwind_tests.patch
new file mode 100644
index 0000000..c11065b
--- /dev/null
+++ b/patches/panic_unwind_tests.patch
@@ -0,0 +1,53 @@
+diff --git a/patches/panic_unwind_tests.patch b/patches/panic_unwind_tests.patch
+index adf431e..e69de29 100644
+--- a/patches/panic_unwind_tests.patch
++++ b/patches/panic_unwind_tests.patch
+@@ -1,24 +0,0 @@
+-diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs
+-index 9aa3484..53ee7d8 100644
+---- a/tests/sync_broadcast.rs
+-+++ b/tests/sync_broadcast.rs
+-@@ -292,6 +292,7 @@ fn capacity_too_big() {
+-
+- #[test]
+-+#[cfg(panic = "unwind")]
+- #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+- fn panic_in_clone() {
+- use std::panic::{self, AssertUnwindSafe};
+-
+-diff --git a/tests/sync_watch.rs b/tests/sync_watch.rs
+-index 34f9b78..e8eacce 100644
+---- a/tests/sync_watch.rs
+-+++ b/tests/sync_watch.rs
+-@@ -214,6 +214,7 @@ fn reopened_after_subscribe() {
+-
+- #[test]
+-+#[cfg(panic = "unwind")]
+- #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+- fn send_modify_panic() {
+- let (tx, mut rx) = watch::channel("one");
+-
+diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs
+index 67c378b..cd66924 100644
+--- a/tests/sync_broadcast.rs
++++ b/tests/sync_broadcast.rs
+@@ -291,6 +291,7 @@ fn capacity_too_big() {
+ }
+
+ #[test]
++#[cfg(panic = "unwind")]
+ #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+ fn panic_in_clone() {
+ use std::panic::{self, AssertUnwindSafe};
+diff --git a/tests/sync_watch.rs b/tests/sync_watch.rs
+index 34f9b78..d4f8ce8 100644
+--- a/tests/sync_watch.rs
++++ b/tests/sync_watch.rs
+@@ -213,6 +213,7 @@ fn reopened_after_subscribe() {
+ }
+
+ #[test]
++#[cfg(panic = "unwind")]
+ #[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+ fn send_modify_panic() {
+ let (tx, mut rx) = watch::channel("one");
diff --git a/patches/sync_broadcast.patch b/patches/sync_broadcast.patch
deleted file mode 100644
index c63a4a7..0000000
--- a/patches/sync_broadcast.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs
-index 5f79800..9ef7927 100644
---- a/tests/sync_broadcast.rs
-+++ b/tests/sync_broadcast.rs
-@@ -286,6 +286,7 @@ fn capacity_too_big() {
- }
-
- #[test]
-+#[cfg(not(target_os = "android"))]
- fn panic_in_clone() {
- use std::panic::{self, AssertUnwindSafe};
-
diff --git a/src/blocking.rs b/src/blocking.rs
index f88b1db..f172399 100644
--- a/src/blocking.rs
+++ b/src/blocking.rs
@@ -1,5 +1,11 @@
cfg_rt! {
pub(crate) use crate::runtime::spawn_blocking;
+
+ cfg_fs! {
+ #[allow(unused_imports)]
+ pub(crate) use crate::runtime::spawn_mandatory_blocking;
+ }
+
pub(crate) use crate::task::JoinHandle;
}
@@ -16,7 +22,16 @@
{
assert_send_sync::<JoinHandle<std::cell::Cell<()>>>();
panic!("requires the `rt` Tokio feature flag")
+ }
+ cfg_fs! {
+ pub(crate) fn spawn_mandatory_blocking<F, R>(_f: F) -> Option<JoinHandle<R>>
+ where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+ {
+ panic!("requires the `rt` Tokio feature flag")
+ }
}
pub(crate) struct JoinHandle<R> {
diff --git a/src/doc/mod.rs b/src/doc/mod.rs
index 3a94934..7992fc6 100644
--- a/src/doc/mod.rs
+++ b/src/doc/mod.rs
@@ -21,4 +21,3 @@
pub enum NotDefinedHere {}
pub mod os;
-pub mod winapi;
diff --git a/src/doc/winapi.rs b/src/doc/winapi.rs
deleted file mode 100644
index be68749..0000000
--- a/src/doc/winapi.rs
+++ /dev/null
@@ -1,66 +0,0 @@
-//! See [winapi].
-//!
-//! [winapi]: https://docs.rs/winapi
-
-/// See [winapi::shared](https://docs.rs/winapi/*/winapi/shared/index.html).
-pub mod shared {
- /// See [winapi::shared::winerror](https://docs.rs/winapi/*/winapi/shared/winerror/index.html).
- #[allow(non_camel_case_types)]
- pub mod winerror {
- /// See [winapi::shared::winerror::ERROR_ACCESS_DENIED][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_ACCESS_DENIED.html
- pub type ERROR_ACCESS_DENIED = crate::doc::NotDefinedHere;
-
- /// See [winapi::shared::winerror::ERROR_PIPE_BUSY][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_PIPE_BUSY.html
- pub type ERROR_PIPE_BUSY = crate::doc::NotDefinedHere;
-
- /// See [winapi::shared::winerror::ERROR_MORE_DATA][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/shared/winerror/constant.ERROR_MORE_DATA.html
- pub type ERROR_MORE_DATA = crate::doc::NotDefinedHere;
- }
-}
-
-/// See [winapi::um](https://docs.rs/winapi/*/winapi/um/index.html).
-pub mod um {
- /// See [winapi::um::winbase](https://docs.rs/winapi/*/winapi/um/winbase/index.html).
- #[allow(non_camel_case_types)]
- pub mod winbase {
- /// See [winapi::um::winbase::PIPE_TYPE_MESSAGE][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_MESSAGE.html
- pub type PIPE_TYPE_MESSAGE = crate::doc::NotDefinedHere;
-
- /// See [winapi::um::winbase::PIPE_TYPE_BYTE][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_TYPE_BYTE.html
- pub type PIPE_TYPE_BYTE = crate::doc::NotDefinedHere;
-
- /// See [winapi::um::winbase::PIPE_CLIENT_END][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_CLIENT_END.html
- pub type PIPE_CLIENT_END = crate::doc::NotDefinedHere;
-
- /// See [winapi::um::winbase::PIPE_SERVER_END][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.PIPE_SERVER_END.html
- pub type PIPE_SERVER_END = crate::doc::NotDefinedHere;
-
- /// See [winapi::um::winbase::SECURITY_IDENTIFICATION][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/um/winbase/constant.SECURITY_IDENTIFICATION.html
- pub type SECURITY_IDENTIFICATION = crate::doc::NotDefinedHere;
- }
-
- /// See [winapi::um::minwinbase](https://docs.rs/winapi/*/winapi/um/minwinbase/index.html).
- #[allow(non_camel_case_types)]
- pub mod minwinbase {
- /// See [winapi::um::minwinbase::SECURITY_ATTRIBUTES][winapi]
- ///
- /// [winapi]: https://docs.rs/winapi/*/winapi/um/minwinbase/constant.SECURITY_ATTRIBUTES.html
- pub type SECURITY_ATTRIBUTES = crate::doc::NotDefinedHere;
- }
-}
diff --git a/src/fs/file.rs b/src/fs/file.rs
index 61071cf..d513beb 100644
--- a/src/fs/file.rs
+++ b/src/fs/file.rs
@@ -20,16 +20,16 @@
use std::task::Poll::*;
#[cfg(test)]
-use super::mocks::spawn_blocking;
-#[cfg(test)]
use super::mocks::JoinHandle;
#[cfg(test)]
use super::mocks::MockFile as StdFile;
-#[cfg(not(test))]
-use crate::blocking::spawn_blocking;
+#[cfg(test)]
+use super::mocks::{spawn_blocking, spawn_mandatory_blocking};
#[cfg(not(test))]
use crate::blocking::JoinHandle;
#[cfg(not(test))]
+use crate::blocking::{spawn_blocking, spawn_mandatory_blocking};
+#[cfg(not(test))]
use std::fs::File as StdFile;
/// A reference to an open file on the filesystem.
@@ -565,29 +565,30 @@
let me = self.get_mut();
let inner = me.inner.get_mut();
- loop {
- match inner.state {
- Busy(_) => panic!("must wait for poll_complete before calling start_seek"),
- Idle(ref mut buf_cell) => {
- let mut buf = buf_cell.take().unwrap();
+ match inner.state {
+ Busy(_) => Err(io::Error::new(
+ io::ErrorKind::Other,
+ "other file operation is pending, call poll_complete before start_seek",
+ )),
+ Idle(ref mut buf_cell) => {
+ let mut buf = buf_cell.take().unwrap();
- // Factor in any unread data from the buf
- if !buf.is_empty() {
- let n = buf.discard_read();
+ // Factor in any unread data from the buf
+ if !buf.is_empty() {
+ let n = buf.discard_read();
- if let SeekFrom::Current(ref mut offset) = pos {
- *offset += n;
- }
+ if let SeekFrom::Current(ref mut offset) = pos {
+ *offset += n;
}
-
- let std = me.std.clone();
-
- inner.state = Busy(spawn_blocking(move || {
- let res = (&*std).seek(pos);
- (Operation::Seek(res), buf)
- }));
- return Ok(());
}
+
+ let std = me.std.clone();
+
+ inner.state = Busy(spawn_blocking(move || {
+ let res = (&*std).seek(pos);
+ (Operation::Seek(res), buf)
+ }));
+ Ok(())
}
}
}
@@ -649,7 +650,7 @@
let n = buf.copy_from(src);
let std = me.std.clone();
- inner.state = Busy(spawn_blocking(move || {
+ let blocking_task_join_handle = spawn_mandatory_blocking(move || {
let res = if let Some(seek) = seek {
(&*std).seek(seek).and_then(|_| buf.write_to(&mut &*std))
} else {
@@ -657,7 +658,12 @@
};
(Operation::Write(res), buf)
- }));
+ })
+ .ok_or_else(|| {
+ io::Error::new(io::ErrorKind::Other, "background task failed")
+ })?;
+
+ inner.state = Busy(blocking_task_join_handle);
return Ready(Ok(n));
}
diff --git a/src/fs/file/tests.rs b/src/fs/file/tests.rs
index 28b5ffe..7c61b3c 100644
--- a/src/fs/file/tests.rs
+++ b/src/fs/file/tests.rs
@@ -228,14 +228,15 @@
}
#[test]
+#[cfg_attr(miri, ignore)] // takes a really long time with miri
fn read_with_buffer_larger_than_max() {
// Chunks
- let chunk_a = 16 * 1024;
+ let chunk_a = crate::io::blocking::MAX_BUF;
let chunk_b = chunk_a * 2;
let chunk_c = chunk_a * 3;
let chunk_d = chunk_a * 4;
- assert_eq!(chunk_d / 1024, 64);
+ assert_eq!(chunk_d / 1024 / 1024, 8);
let mut data = vec![];
for i in 0..(chunk_d - 1) {
@@ -299,14 +300,15 @@
}
#[test]
+#[cfg_attr(miri, ignore)] // takes a really long time with miri
fn write_with_buffer_larger_than_max() {
// Chunks
- let chunk_a = 16 * 1024;
+ let chunk_a = crate::io::blocking::MAX_BUF;
let chunk_b = chunk_a * 2;
let chunk_c = chunk_a * 3;
let chunk_d = chunk_a * 4;
- assert_eq!(chunk_d / 1024, 64);
+ assert_eq!(chunk_d / 1024 / 1024, 8);
let mut data = vec![];
for i in 0..(chunk_d - 1) {
@@ -953,3 +955,24 @@
assert_eq!(n, FOO.len());
assert_eq!(&buf[..n], FOO);
}
+
+#[test]
+fn busy_file_seek_error() {
+ let mut file = MockFile::default();
+ let mut seq = Sequence::new();
+ file.expect_inner_write()
+ .once()
+ .in_sequence(&mut seq)
+ .returning(|_| Err(io::ErrorKind::Other.into()));
+
+ let mut file = crate::io::BufReader::new(File::from_std(file));
+ {
+ let mut t = task::spawn(file.write(HELLO));
+ assert_ready_ok!(t.poll());
+ }
+
+ pool::run_one();
+
+ let mut t = task::spawn(file.seek(SeekFrom::Start(0)));
+ assert_ready_err!(t.poll());
+}
diff --git a/src/fs/mocks.rs b/src/fs/mocks.rs
index 68ef4f3..aa01e24 100644
--- a/src/fs/mocks.rs
+++ b/src/fs/mocks.rs
@@ -81,7 +81,7 @@
}
}
-thread_local! {
+tokio_thread_local! {
static QUEUE: RefCell<VecDeque<Box<dyn FnOnce() + Send>>> = RefCell::new(VecDeque::new())
}
@@ -105,6 +105,21 @@
JoinHandle { rx }
}
+pub(super) fn spawn_mandatory_blocking<F, R>(f: F) -> Option<JoinHandle<R>>
+where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+{
+ let (tx, rx) = oneshot::channel();
+ let task = Box::new(move || {
+ let _ = tx.send(f());
+ });
+
+ QUEUE.with(|cell| cell.borrow_mut().push_back(task));
+
+ Some(JoinHandle { rx })
+}
+
impl<T> Future for JoinHandle<T> {
type Output = Result<T, io::Error>;
diff --git a/src/fs/mod.rs b/src/fs/mod.rs
index ca0264b..3afefc6 100644
--- a/src/fs/mod.rs
+++ b/src/fs/mod.rs
@@ -22,6 +22,24 @@
//! `std::io::ErrorKind::WouldBlock` if a *worker* thread can not be converted
//! to a *backup* thread immediately.
//!
+//! **Warning**: These adapters may create a large number of temporary tasks,
+//! especially when reading large files. When performing a lot of operations
+//! in one batch, it may be significantly faster to use [`spawn_blocking`]
+//! directly:
+//!
+//! ```
+//! use tokio::fs::File;
+//! use std::io::{BufReader, BufRead};
+//! async fn count_lines(file: File) -> Result<usize, std::io::Error> {
+//! let file = file.into_std().await;
+//! tokio::task::spawn_blocking(move || {
+//! let line_count = BufReader::new(file).lines().count();
+//! Ok(line_count)
+//! }).await?
+//! }
+//! ```
+//!
+//! [`spawn_blocking`]: fn@crate::task::spawn_blocking
//! [`AsyncRead`]: trait@crate::io::AsyncRead
mod canonicalize;
diff --git a/src/fs/open_options.rs b/src/fs/open_options.rs
index f3b4654..c600ef5 100644
--- a/src/fs/open_options.rs
+++ b/src/fs/open_options.rs
@@ -542,7 +542,7 @@
/// # Examples
///
/// ```no_run
- /// use winapi::um::winbase::FILE_FLAG_DELETE_ON_CLOSE;
+ /// use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_DELETE_ON_CLOSE;
/// use tokio::fs::OpenOptions;
///
/// # #[tokio::main]
@@ -581,7 +581,7 @@
/// # Examples
///
/// ```no_run
- /// use winapi::um::winnt::FILE_ATTRIBUTE_HIDDEN;
+ /// use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN;
/// use tokio::fs::OpenOptions;
///
/// # #[tokio::main]
@@ -624,7 +624,7 @@
/// # Examples
///
/// ```no_run
- /// use winapi::um::winbase::SECURITY_IDENTIFICATION;
+ /// use windows_sys::Win32::Storage::FileSystem::SECURITY_IDENTIFICATION;
/// use tokio::fs::OpenOptions;
///
/// # #[tokio::main]
diff --git a/src/fs/open_options/mock_open_options.rs b/src/fs/open_options/mock_open_options.rs
index cbbda0e..17b4a48 100644
--- a/src/fs/open_options/mock_open_options.rs
+++ b/src/fs/open_options/mock_open_options.rs
@@ -1,3 +1,4 @@
+#![allow(unreachable_pub)]
//! Mock version of std::fs::OpenOptions;
use mockall::mock;
diff --git a/src/fs/read_dir.rs b/src/fs/read_dir.rs
index 281ea4c..9471e8c 100644
--- a/src/fs/read_dir.rs
+++ b/src/fs/read_dir.rs
@@ -1,5 +1,6 @@
use crate::fs::asyncify;
+use std::collections::VecDeque;
use std::ffi::OsString;
use std::fs::{FileType, Metadata};
use std::future::Future;
@@ -19,6 +20,8 @@
#[cfg(not(test))]
use crate::blocking::JoinHandle;
+const CHUNK_SIZE: usize = 32;
+
/// Returns a stream over the entries within a directory.
///
/// This is an async version of [`std::fs::read_dir`](std::fs::read_dir)
@@ -29,12 +32,17 @@
/// [`spawn_blocking`]: crate::task::spawn_blocking
pub async fn read_dir(path: impl AsRef<Path>) -> io::Result<ReadDir> {
let path = path.as_ref().to_owned();
- let std = asyncify(|| std::fs::read_dir(path)).await?;
+ asyncify(|| -> io::Result<ReadDir> {
+ let mut std = std::fs::read_dir(path)?;
+ let mut buf = VecDeque::with_capacity(CHUNK_SIZE);
+ ReadDir::next_chunk(&mut buf, &mut std);
- Ok(ReadDir(State::Idle(Some(std))))
+ Ok(ReadDir(State::Idle(Some((buf, std)))))
+ })
+ .await
}
-/// Reads the the entries in a directory.
+/// Reads the entries in a directory.
///
/// This struct is returned from the [`read_dir`] function of this module and
/// will yield instances of [`DirEntry`]. Through a [`DirEntry`] information
@@ -58,8 +66,8 @@
#[derive(Debug)]
enum State {
- Idle(Option<std::fs::ReadDir>),
- Pending(JoinHandle<(Option<io::Result<std::fs::DirEntry>>, std::fs::ReadDir)>),
+ Idle(Option<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir)>),
+ Pending(JoinHandle<(VecDeque<io::Result<DirEntry>>, std::fs::ReadDir)>),
}
impl ReadDir {
@@ -94,29 +102,57 @@
pub fn poll_next_entry(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<DirEntry>>> {
loop {
match self.0 {
- State::Idle(ref mut std) => {
- let mut std = std.take().unwrap();
+ State::Idle(ref mut data) => {
+ let (buf, _) = data.as_mut().unwrap();
+
+ if let Some(ent) = buf.pop_front() {
+ return Poll::Ready(ent.map(Some));
+ };
+
+ let (mut buf, mut std) = data.take().unwrap();
self.0 = State::Pending(spawn_blocking(move || {
- let ret = std.next();
- (ret, std)
+ ReadDir::next_chunk(&mut buf, &mut std);
+ (buf, std)
}));
}
State::Pending(ref mut rx) => {
- let (ret, std) = ready!(Pin::new(rx).poll(cx))?;
- self.0 = State::Idle(Some(std));
+ let (mut buf, std) = ready!(Pin::new(rx).poll(cx))?;
- let ret = match ret {
- Some(Ok(std)) => Ok(Some(DirEntry(Arc::new(std)))),
+ let ret = match buf.pop_front() {
+ Some(Ok(x)) => Ok(Some(x)),
Some(Err(e)) => Err(e),
None => Ok(None),
};
+ self.0 = State::Idle(Some((buf, std)));
+
return Poll::Ready(ret);
}
}
}
}
+
+ fn next_chunk(buf: &mut VecDeque<io::Result<DirEntry>>, std: &mut std::fs::ReadDir) {
+ for ret in std.by_ref().take(CHUNK_SIZE) {
+ let success = ret.is_ok();
+
+ buf.push_back(ret.map(|std| DirEntry {
+ #[cfg(not(any(
+ target_os = "solaris",
+ target_os = "illumos",
+ target_os = "haiku",
+ target_os = "vxworks"
+ )))]
+ file_type: std.file_type().ok(),
+ std: Arc::new(std),
+ }));
+
+ if !success {
+ break;
+ }
+ }
+ }
}
feature! {
@@ -160,7 +196,16 @@
/// filesystem. Each entry can be inspected via methods to learn about the full
/// path or possibly other metadata through per-platform extension traits.
#[derive(Debug)]
-pub struct DirEntry(Arc<std::fs::DirEntry>);
+pub struct DirEntry {
+ #[cfg(not(any(
+ target_os = "solaris",
+ target_os = "illumos",
+ target_os = "haiku",
+ target_os = "vxworks"
+ )))]
+ file_type: Option<FileType>,
+ std: Arc<std::fs::DirEntry>,
+}
impl DirEntry {
/// Returns the full path to the file that this entry represents.
@@ -193,7 +238,7 @@
///
/// The exact text, of course, depends on what files you have in `.`.
pub fn path(&self) -> PathBuf {
- self.0.path()
+ self.std.path()
}
/// Returns the bare file name of this directory entry without any other
@@ -214,7 +259,7 @@
/// # }
/// ```
pub fn file_name(&self) -> OsString {
- self.0.file_name()
+ self.std.file_name()
}
/// Returns the metadata for the file that this entry points at.
@@ -248,7 +293,7 @@
/// # }
/// ```
pub async fn metadata(&self) -> io::Result<Metadata> {
- let std = self.0.clone();
+ let std = self.std.clone();
asyncify(move || std.metadata()).await
}
@@ -283,13 +328,23 @@
/// # }
/// ```
pub async fn file_type(&self) -> io::Result<FileType> {
- let std = self.0.clone();
+ #[cfg(not(any(
+ target_os = "solaris",
+ target_os = "illumos",
+ target_os = "haiku",
+ target_os = "vxworks"
+ )))]
+ if let Some(file_type) = self.file_type {
+ return Ok(file_type);
+ }
+
+ let std = self.std.clone();
asyncify(move || std.file_type()).await
}
/// Returns a reference to the underlying `std::fs::DirEntry`.
#[cfg(unix)]
pub(super) fn as_inner(&self) -> &std::fs::DirEntry {
- &self.0
+ &self.std
}
}
diff --git a/src/future/block_on.rs b/src/future/block_on.rs
index 91f9cc0..2c2ab37 100644
--- a/src/future/block_on.rs
+++ b/src/future/block_on.rs
@@ -1,15 +1,22 @@
use std::future::Future;
cfg_rt! {
+ #[track_caller]
pub(crate) fn block_on<F: Future>(f: F) -> F::Output {
- let mut e = crate::runtime::enter::enter(false);
+ let mut e = crate::runtime::context::try_enter_blocking_region().expect(
+ "Cannot block the current thread from within a runtime. This \
+ happens because a function attempted to block the current \
+ thread while the thread is being used to drive asynchronous \
+ tasks."
+ );
e.block_on(f).unwrap()
}
}
cfg_not_rt! {
+ #[track_caller]
pub(crate) fn block_on<F: Future>(f: F) -> F::Output {
- let mut park = crate::park::thread::CachedParkThread::new();
+ let mut park = crate::runtime::park::CachedParkThread::new();
park.block_on(f).unwrap()
}
}
diff --git a/src/future/mod.rs b/src/future/mod.rs
index 96483ac..084ddc5 100644
--- a/src/future/mod.rs
+++ b/src/future/mod.rs
@@ -8,11 +8,6 @@
mod poll_fn;
pub use poll_fn::poll_fn;
-cfg_not_loom! {
- mod ready;
- pub(crate) use ready::{ok, Ready};
-}
-
cfg_process! {
mod try_join;
pub(crate) use try_join::try_join3;
diff --git a/src/future/poll_fn.rs b/src/future/poll_fn.rs
index d82ce89..074d943 100644
--- a/src/future/poll_fn.rs
+++ b/src/future/poll_fn.rs
@@ -7,13 +7,23 @@
use std::pin::Pin;
use std::task::{Context, Poll};
+// This struct is intentionally `!Unpin` when `F` is `!Unpin`. This is to
+// mitigate the issue where rust puts noalias on mutable references to the
+// `PollFn` type if it is `Unpin`. If the closure has ownership of a future,
+// then this "leaks" and the future is affected by noalias too, which we don't
+// want.
+//
+// See this thread for more information:
+// <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484>
+//
+// The fact that `PollFn` is not `Unpin` when it shouldn't be is tested in
+// `tests/async_send_sync.rs`.
+
/// Future for the [`poll_fn`] function.
pub struct PollFn<F> {
f: F,
}
-impl<F> Unpin for PollFn<F> {}
-
/// Creates a new future wrapping around a function returning [`Poll`].
pub fn poll_fn<T, F>(f: F) -> PollFn<F>
where
@@ -34,7 +44,17 @@
{
type Output = T;
- fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<T> {
- (&mut self.f)(cx)
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<T> {
+ // Safety: We never construct a `Pin<&mut F>` anywhere, so accessing `f`
+ // mutably in an unpinned way is sound.
+ //
+ // This use of unsafe cannot be replaced with the pin-project macro
+ // because:
+ // * If we put `#[pin]` on the field, then it gives us a `Pin<&mut F>`,
+ // which we can't use to call the closure.
+ // * If we don't put `#[pin]` on the field, then it makes `PollFn` be
+ // unconditionally `Unpin`, which we also don't want.
+ let me = unsafe { Pin::into_inner_unchecked(self) };
+ (me.f)(cx)
}
}
diff --git a/src/future/ready.rs b/src/future/ready.rs
deleted file mode 100644
index de2d60c..0000000
--- a/src/future/ready.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-use std::future::Future;
-use std::pin::Pin;
-use std::task::{Context, Poll};
-
-/// Future for the [`ok`](ok()) function.
-///
-/// `pub` in order to use the future as an associated type in a sealed trait.
-#[derive(Debug)]
-// Used as an associated type in a "sealed" trait.
-#[allow(unreachable_pub)]
-pub struct Ready<T>(Option<T>);
-
-impl<T> Unpin for Ready<T> {}
-
-impl<T> Future for Ready<T> {
- type Output = T;
-
- #[inline]
- fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<T> {
- Poll::Ready(self.0.take().unwrap())
- }
-}
-
-/// Creates a future that is immediately ready with a success value.
-pub(crate) fn ok<T, E>(t: T) -> Ready<Result<T, E>> {
- Ready(Some(Ok(t)))
-}
diff --git a/src/io/async_fd.rs b/src/io/async_fd.rs
index 9ec5b7f..92fc6b3 100644
--- a/src/io/async_fd.rs
+++ b/src/io/async_fd.rs
@@ -1,4 +1,6 @@
-use crate::io::driver::{Handle, Interest, ReadyEvent, Registration};
+use crate::io::Interest;
+use crate::runtime::io::{ReadyEvent, Registration};
+use crate::runtime::scheduler;
use mio::unix::SourceFd;
use std::io;
@@ -81,6 +83,7 @@
///
/// impl AsyncTcpStream {
/// pub fn new(tcp: TcpStream) -> io::Result<Self> {
+/// tcp.set_nonblocking(true)?;
/// Ok(Self {
/// inner: AsyncFd::new(tcp)?,
/// })
@@ -166,12 +169,18 @@
const ALL_INTEREST: Interest = Interest::READABLE.add(Interest::WRITABLE);
impl<T: AsRawFd> AsyncFd<T> {
- #[inline]
/// Creates an AsyncFd backed by (and taking ownership of) an object
/// implementing [`AsRawFd`]. The backing file descriptor is cached at the
/// time of creation.
///
/// This method must be called in the context of a tokio runtime.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if there is no current reactor set, or if the `rt`
+ /// feature flag is not enabled.
+ #[inline]
+ #[track_caller]
pub fn new(inner: T) -> io::Result<Self>
where
T: AsRawFd,
@@ -179,19 +188,26 @@
Self::with_interest(inner, ALL_INTEREST)
}
- #[inline]
/// Creates new instance as `new` with additional ability to customize interest,
/// allowing to specify whether file descriptor will be polled for read, write or both.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if there is no current reactor set, or if the `rt`
+ /// feature flag is not enabled.
+ #[inline]
+ #[track_caller]
pub fn with_interest(inner: T, interest: Interest) -> io::Result<Self>
where
T: AsRawFd,
{
- Self::new_with_handle_and_interest(inner, Handle::current(), interest)
+ Self::new_with_handle_and_interest(inner, scheduler::Handle::current(), interest)
}
+ #[track_caller]
pub(crate) fn new_with_handle_and_interest(
inner: T,
- handle: Handle,
+ handle: scheduler::Handle,
interest: Interest,
) -> io::Result<Self> {
let fd = inner.as_raw_fd();
@@ -525,7 +541,7 @@
#[cfg_attr(docsrs, doc(alias = "with_io"))]
pub fn try_io<R>(
&mut self,
- f: impl FnOnce(&AsyncFd<Inner>) -> io::Result<R>,
+ f: impl FnOnce(&'a AsyncFd<Inner>) -> io::Result<R>,
) -> Result<io::Result<R>, TryIoError> {
let result = f(self.async_fd);
@@ -542,12 +558,12 @@
}
/// Returns a shared reference to the inner [`AsyncFd`].
- pub fn get_ref(&self) -> &AsyncFd<Inner> {
+ pub fn get_ref(&self) -> &'a AsyncFd<Inner> {
self.async_fd
}
/// Returns a shared reference to the backing object of the inner [`AsyncFd`].
- pub fn get_inner(&self) -> &Inner {
+ pub fn get_inner(&self) -> &'a Inner {
self.get_ref().get_ref()
}
}
@@ -598,7 +614,7 @@
&mut self,
f: impl FnOnce(&mut AsyncFd<Inner>) -> io::Result<R>,
) -> Result<io::Result<R>, TryIoError> {
- let result = f(&mut self.async_fd);
+ let result = f(self.async_fd);
if let Err(e) = result.as_ref() {
if e.kind() == io::ErrorKind::WouldBlock {
diff --git a/src/io/blocking.rs b/src/io/blocking.rs
index 1d79ee7..416573e 100644
--- a/src/io/blocking.rs
+++ b/src/io/blocking.rs
@@ -26,7 +26,7 @@
pos: usize,
}
-pub(crate) const MAX_BUF: usize = 16 * 1024;
+pub(crate) const MAX_BUF: usize = 2 * 1024 * 1024;
#[derive(Debug)]
enum State<T> {
@@ -34,8 +34,9 @@
Busy(sys::Blocking<(io::Result<usize>, Buf, T)>),
}
-cfg_io_std! {
+cfg_io_blocking! {
impl<T> Blocking<T> {
+ #[cfg_attr(feature = "fs", allow(dead_code))]
pub(crate) fn new(inner: T) -> Blocking<T> {
Blocking {
inner: Some(inner),
diff --git a/src/io/bsd/poll_aio.rs b/src/io/bsd/poll_aio.rs
index f1ac4b2..6ac9e28 100644
--- a/src/io/bsd/poll_aio.rs
+++ b/src/io/bsd/poll_aio.rs
@@ -1,6 +1,8 @@
//! Use POSIX AIO futures with Tokio.
-use crate::io::driver::{Handle, Interest, ReadyEvent, Registration};
+use crate::io::interest::Interest;
+use crate::runtime::io::{ReadyEvent, Registration};
+use crate::runtime::scheduler;
use mio::event::Source;
use mio::Registry;
use mio::Token;
@@ -117,7 +119,7 @@
fn new_with_interest(io: E, interest: Interest) -> io::Result<Self> {
let mut io = MioSource(io);
- let handle = Handle::current();
+ let handle = scheduler::Handle::current();
let registration = Registration::new_with_interest_and_handle(&mut io, interest, handle)?;
Ok(Self { io, registration })
}
diff --git a/src/io/driver/mod.rs b/src/io/driver/mod.rs
deleted file mode 100644
index 19f67a2..0000000
--- a/src/io/driver/mod.rs
+++ /dev/null
@@ -1,354 +0,0 @@
-#![cfg_attr(not(feature = "rt"), allow(dead_code))]
-
-mod interest;
-#[allow(unreachable_pub)]
-pub use interest::Interest;
-
-mod ready;
-#[allow(unreachable_pub)]
-pub use ready::Ready;
-
-mod registration;
-pub(crate) use registration::Registration;
-
-mod scheduled_io;
-use scheduled_io::ScheduledIo;
-
-use crate::park::{Park, Unpark};
-use crate::util::slab::{self, Slab};
-use crate::{loom::sync::Mutex, util::bit};
-
-use std::fmt;
-use std::io;
-use std::sync::{Arc, Weak};
-use std::time::Duration;
-
-/// I/O driver, backed by Mio.
-pub(crate) struct Driver {
- /// Tracks the number of times `turn` is called. It is safe for this to wrap
- /// as it is mostly used to determine when to call `compact()`.
- tick: u8,
-
- /// Reuse the `mio::Events` value across calls to poll.
- events: Option<mio::Events>,
-
- /// Primary slab handle containing the state for each resource registered
- /// with this driver. During Drop this is moved into the Inner structure, so
- /// this is an Option to allow it to be vacated (until Drop this is always
- /// Some).
- resources: Option<Slab<ScheduledIo>>,
-
- /// The system event queue.
- poll: mio::Poll,
-
- /// State shared between the reactor and the handles.
- inner: Arc<Inner>,
-}
-
-/// A reference to an I/O driver.
-#[derive(Clone)]
-pub(crate) struct Handle {
- inner: Weak<Inner>,
-}
-
-#[derive(Debug)]
-pub(crate) struct ReadyEvent {
- tick: u8,
- pub(crate) ready: Ready,
-}
-
-pub(super) struct Inner {
- /// Primary slab handle containing the state for each resource registered
- /// with this driver.
- ///
- /// The ownership of this slab is moved into this structure during
- /// `Driver::drop`, so that `Inner::drop` can notify all outstanding handles
- /// without risking new ones being registered in the meantime.
- resources: Mutex<Option<Slab<ScheduledIo>>>,
-
- /// Registers I/O resources.
- registry: mio::Registry,
-
- /// Allocates `ScheduledIo` handles when creating new resources.
- pub(super) io_dispatch: slab::Allocator<ScheduledIo>,
-
- /// Used to wake up the reactor from a call to `turn`.
- waker: mio::Waker,
-}
-
-#[derive(Debug, Eq, PartialEq, Clone, Copy)]
-enum Direction {
- Read,
- Write,
-}
-
-enum Tick {
- Set(u8),
- Clear(u8),
-}
-
-// TODO: Don't use a fake token. Instead, reserve a slot entry for the wakeup
-// token.
-const TOKEN_WAKEUP: mio::Token = mio::Token(1 << 31);
-
-const ADDRESS: bit::Pack = bit::Pack::least_significant(24);
-
-// Packs the generation value in the `readiness` field.
-//
-// The generation prevents a race condition where a slab slot is reused for a
-// new socket while the I/O driver is about to apply a readiness event. The
-// generation value is checked when setting new readiness. If the generation do
-// not match, then the readiness event is discarded.
-const GENERATION: bit::Pack = ADDRESS.then(7);
-
-fn _assert_kinds() {
- fn _assert<T: Send + Sync>() {}
-
- _assert::<Handle>();
-}
-
-// ===== impl Driver =====
-
-impl Driver {
- /// Creates a new event loop, returning any error that happened during the
- /// creation.
- pub(crate) fn new() -> io::Result<Driver> {
- let poll = mio::Poll::new()?;
- let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
- let registry = poll.registry().try_clone()?;
-
- let slab = Slab::new();
- let allocator = slab.allocator();
-
- Ok(Driver {
- tick: 0,
- events: Some(mio::Events::with_capacity(1024)),
- poll,
- resources: Some(slab),
- inner: Arc::new(Inner {
- resources: Mutex::new(None),
- registry,
- io_dispatch: allocator,
- waker,
- }),
- })
- }
-
- /// Returns a handle to this event loop which can be sent across threads
- /// and can be used as a proxy to the event loop itself.
- ///
- /// Handles are cloneable and clones always refer to the same event loop.
- /// This handle is typically passed into functions that create I/O objects
- /// to bind them to this event loop.
- pub(crate) fn handle(&self) -> Handle {
- Handle {
- inner: Arc::downgrade(&self.inner),
- }
- }
-
- fn turn(&mut self, max_wait: Option<Duration>) -> io::Result<()> {
- // How often to call `compact()` on the resource slab
- const COMPACT_INTERVAL: u8 = 255;
-
- self.tick = self.tick.wrapping_add(1);
-
- if self.tick == COMPACT_INTERVAL {
- self.resources.as_mut().unwrap().compact()
- }
-
- let mut events = self.events.take().expect("i/o driver event store missing");
-
- // Block waiting for an event to happen, peeling out how many events
- // happened.
- match self.poll.poll(&mut events, max_wait) {
- Ok(_) => {}
- Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
- Err(e) => return Err(e),
- }
-
- // Process all the events that came in, dispatching appropriately
- for event in events.iter() {
- let token = event.token();
-
- if token != TOKEN_WAKEUP {
- self.dispatch(token, Ready::from_mio(event));
- }
- }
-
- self.events = Some(events);
-
- Ok(())
- }
-
- fn dispatch(&mut self, token: mio::Token, ready: Ready) {
- let addr = slab::Address::from_usize(ADDRESS.unpack(token.0));
-
- let resources = self.resources.as_mut().unwrap();
-
- let io = match resources.get(addr) {
- Some(io) => io,
- None => return,
- };
-
- let res = io.set_readiness(Some(token.0), Tick::Set(self.tick), |curr| curr | ready);
-
- if res.is_err() {
- // token no longer valid!
- return;
- }
-
- io.wake(ready);
- }
-}
-
-impl Drop for Driver {
- fn drop(&mut self) {
- (*self.inner.resources.lock()) = self.resources.take();
- }
-}
-
-impl Drop for Inner {
- fn drop(&mut self) {
- let resources = self.resources.lock().take();
-
- if let Some(mut slab) = resources {
- slab.for_each(|io| {
- // If a task is waiting on the I/O resource, notify it. The task
- // will then attempt to use the I/O resource and fail due to the
- // driver being shutdown.
- io.shutdown();
- });
- }
- }
-}
-
-impl Park for Driver {
- type Unpark = Handle;
- type Error = io::Error;
-
- fn unpark(&self) -> Self::Unpark {
- self.handle()
- }
-
- fn park(&mut self) -> io::Result<()> {
- self.turn(None)?;
- Ok(())
- }
-
- fn park_timeout(&mut self, duration: Duration) -> io::Result<()> {
- self.turn(Some(duration))?;
- Ok(())
- }
-
- fn shutdown(&mut self) {}
-}
-
-impl fmt::Debug for Driver {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(f, "Driver")
- }
-}
-
-// ===== impl Handle =====
-
-cfg_rt! {
- impl Handle {
- /// Returns a handle to the current reactor.
- ///
- /// # Panics
- ///
- /// This function panics if there is no current reactor set and `rt` feature
- /// flag is not enabled.
- pub(super) fn current() -> Self {
- crate::runtime::context::io_handle().expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.")
- }
- }
-}
-
-cfg_not_rt! {
- impl Handle {
- /// Returns a handle to the current reactor.
- ///
- /// # Panics
- ///
- /// This function panics if there is no current reactor set, or if the `rt`
- /// feature flag is not enabled.
- pub(super) fn current() -> Self {
- panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR)
- }
- }
-}
-
-impl Handle {
- /// Forces a reactor blocked in a call to `turn` to wakeup, or otherwise
- /// makes the next call to `turn` return immediately.
- ///
- /// This method is intended to be used in situations where a notification
- /// needs to otherwise be sent to the main reactor. If the reactor is
- /// currently blocked inside of `turn` then it will wake up and soon return
- /// after this method has been called. If the reactor is not currently
- /// blocked in `turn`, then the next call to `turn` will not block and
- /// return immediately.
- fn wakeup(&self) {
- if let Some(inner) = self.inner() {
- inner.waker.wake().expect("failed to wake I/O driver");
- }
- }
-
- pub(super) fn inner(&self) -> Option<Arc<Inner>> {
- self.inner.upgrade()
- }
-}
-
-impl Unpark for Handle {
- fn unpark(&self) {
- self.wakeup();
- }
-}
-
-impl fmt::Debug for Handle {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(f, "Handle")
- }
-}
-
-// ===== impl Inner =====
-
-impl Inner {
- /// Registers an I/O resource with the reactor for a given `mio::Ready` state.
- ///
- /// The registration token is returned.
- pub(super) fn add_source(
- &self,
- source: &mut impl mio::event::Source,
- interest: Interest,
- ) -> io::Result<slab::Ref<ScheduledIo>> {
- let (address, shared) = self.io_dispatch.allocate().ok_or_else(|| {
- io::Error::new(
- io::ErrorKind::Other,
- "reactor at max registered I/O resources",
- )
- })?;
-
- let token = GENERATION.pack(shared.generation(), ADDRESS.pack(address.as_usize(), 0));
-
- self.registry
- .register(source, mio::Token(token), interest.to_mio())?;
-
- Ok(shared)
- }
-
- /// Deregisters an I/O resource from the reactor.
- pub(super) fn deregister_source(&self, source: &mut impl mio::event::Source) -> io::Result<()> {
- self.registry.deregister(source)
- }
-}
-
-impl Direction {
- pub(super) fn mask(self) -> Ready {
- match self {
- Direction::Read => Ready::READABLE | Ready::READ_CLOSED,
- Direction::Write => Ready::WRITABLE | Ready::WRITE_CLOSED,
- }
- }
-}
diff --git a/src/io/driver/interest.rs b/src/io/interest.rs
similarity index 97%
rename from src/io/driver/interest.rs
rename to src/io/interest.rs
index d6b46df..013c114 100644
--- a/src/io/driver/interest.rs
+++ b/src/io/interest.rs
@@ -1,6 +1,6 @@
#![cfg_attr(not(feature = "net"), allow(dead_code, unreachable_pub))]
-use crate::io::driver::Ready;
+use crate::io::ready::Ready;
use std::fmt;
use std::ops;
@@ -100,7 +100,7 @@
self.0
}
- pub(super) fn mask(self) -> Ready {
+ pub(crate) fn mask(self) -> Ready {
match self {
Interest::READABLE => Ready::READABLE | Ready::READ_CLOSED,
Interest::WRITABLE => Ready::WRITABLE | Ready::WRITE_CLOSED,
diff --git a/src/io/mod.rs b/src/io/mod.rs
index cfdda61..f48035a 100644
--- a/src/io/mod.rs
+++ b/src/io/mod.rs
@@ -1,5 +1,3 @@
-#![cfg_attr(loom, allow(dead_code, unreachable_pub))]
-
//! Traits, helpers, and type definitions for asynchronous I/O functionality.
//!
//! This module is the asynchronous version of `std::io`. Primarily, it
@@ -180,6 +178,12 @@
//! [`Sink`]: https://docs.rs/futures/0.3/futures/sink/trait.Sink.html
//! [`Stream`]: https://docs.rs/futures/0.3/futures/stream/trait.Stream.html
//! [`Write`]: std::io::Write
+
+#![cfg_attr(
+ not(all(feature = "rt", feature = "net")),
+ allow(dead_code, unused_imports)
+)]
+
cfg_io_blocking! {
pub(crate) mod blocking;
}
@@ -205,15 +209,19 @@
pub use std::io::{Error, ErrorKind, Result, SeekFrom};
cfg_io_driver_impl! {
- pub(crate) mod driver;
+ pub(crate) mod interest;
+ pub(crate) mod ready;
cfg_net! {
- pub use driver::{Interest, Ready};
+ pub use interest::Interest;
+ pub use ready::Ready;
}
+ #[cfg_attr(tokio_wasi, allow(unused_imports))]
mod poll_evented;
#[cfg(not(loom))]
+ #[cfg_attr(tokio_wasi, allow(unused_imports))]
pub(crate) use poll_evented::PollEvented;
}
diff --git a/src/io/poll_evented.rs b/src/io/poll_evented.rs
index 44e68a2..dfe9ae3 100644
--- a/src/io/poll_evented.rs
+++ b/src/io/poll_evented.rs
@@ -1,16 +1,19 @@
-use crate::io::driver::{Handle, Interest, Registration};
+use crate::io::interest::Interest;
+use crate::runtime::io::Registration;
+use crate::runtime::scheduler;
use mio::event::Source;
use std::fmt;
use std::io;
use std::ops::Deref;
+use std::panic::{RefUnwindSafe, UnwindSafe};
cfg_io_driver! {
/// Associates an I/O resource that implements the [`std::io::Read`] and/or
/// [`std::io::Write`] traits with the reactor that drives it.
///
/// `PollEvented` uses [`Registration`] internally to take a type that
- /// implements [`mio::event::Source`] as well as [`std::io::Read`] and or
+ /// implements [`mio::event::Source`] as well as [`std::io::Read`] and/or
/// [`std::io::Write`] and associate it with a reactor that will drive it.
///
/// Once the [`mio::event::Source`] type is wrapped by `PollEvented`, it can be
@@ -40,12 +43,12 @@
/// [`poll_read_ready`] again will also indicate read readiness.
///
/// When the operation is attempted and is unable to succeed due to the I/O
- /// resource not being ready, the caller must call `clear_readiness`.
+ /// resource not being ready, the caller must call [`clear_readiness`].
/// This clears the readiness state until a new readiness event is received.
///
/// This allows the caller to implement additional functions. For example,
/// [`TcpListener`] implements poll_accept by using [`poll_read_ready`] and
- /// `clear_read_ready`.
+ /// [`clear_readiness`].
///
/// ## Platform-specific events
///
@@ -56,6 +59,7 @@
/// [`AsyncRead`]: crate::io::AsyncRead
/// [`AsyncWrite`]: crate::io::AsyncWrite
/// [`TcpListener`]: crate::net::TcpListener
+ /// [`clear_readiness`]: Registration::clear_readiness
/// [`poll_read_ready`]: Registration::poll_read_ready
/// [`poll_write_ready`]: Registration::poll_write_ready
pub(crate) struct PollEvented<E: Source> {
@@ -76,6 +80,7 @@
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+ #[track_caller]
#[cfg_attr(feature = "signal", allow(unused))]
pub(crate) fn new(io: E) -> io::Result<Self> {
PollEvented::new_with_interest(io, Interest::READABLE | Interest::WRITABLE)
@@ -96,15 +101,17 @@
/// a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter)
/// function.
+ #[track_caller]
#[cfg_attr(feature = "signal", allow(unused))]
pub(crate) fn new_with_interest(io: E, interest: Interest) -> io::Result<Self> {
- Self::new_with_interest_and_handle(io, interest, Handle::current())
+ Self::new_with_interest_and_handle(io, interest, scheduler::Handle::current())
}
+ #[track_caller]
pub(crate) fn new_with_interest_and_handle(
mut io: E,
interest: Interest,
- handle: Handle,
+ handle: scheduler::Handle,
) -> io::Result<Self> {
let registration = Registration::new_with_interest_and_handle(&mut io, interest, handle)?;
Ok(Self {
@@ -114,11 +121,7 @@
}
/// Returns a reference to the registration.
- #[cfg(any(
- feature = "net",
- all(unix, feature = "process"),
- all(unix, feature = "signal"),
- ))]
+ #[cfg(any(feature = "net"))]
pub(crate) fn registration(&self) -> &Registration {
&self.registration
}
@@ -133,7 +136,7 @@
}
feature! {
- #![any(feature = "net", feature = "process")]
+ #![any(feature = "net", all(unix, feature = "process"))]
use crate::io::ReadBuf;
use std::task::{Context, Poll};
@@ -150,16 +153,32 @@
{
use std::io::Read;
- let n = ready!(self.registration.poll_read_io(cx, || {
- let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]);
- self.io.as_ref().unwrap().read(b)
- }))?;
+ loop {
+ let evt = ready!(self.registration.poll_read_ready(cx))?;
- // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the
- // buffer.
- buf.assume_init(n);
- buf.advance(n);
- Poll::Ready(Ok(()))
+ let b = &mut *(buf.unfilled_mut() as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]);
+ let len = b.len();
+
+ match self.io.as_ref().unwrap().read(b) {
+ Ok(n) => {
+ // if we read a partially full buffer, this is sufficient on unix to show
+ // that the socket buffer has been drained
+ if n > 0 && (!cfg!(windows) && n < len) {
+ self.registration.clear_readiness(evt);
+ }
+
+ // Safety: We trust `TcpStream::read` to have filled up `n` bytes in the
+ // buffer.
+ buf.assume_init(n);
+ buf.advance(n);
+ return Poll::Ready(Ok(()));
+ },
+ Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+ self.registration.clear_readiness(evt);
+ }
+ Err(e) => return Poll::Ready(Err(e)),
+ }
+ }
}
pub(crate) fn poll_write<'a>(&'a self, cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>>
@@ -167,10 +186,29 @@
&'a E: io::Write + 'a,
{
use std::io::Write;
- self.registration.poll_write_io(cx, || self.io.as_ref().unwrap().write(buf))
+
+ loop {
+ let evt = ready!(self.registration.poll_write_ready(cx))?;
+
+ match self.io.as_ref().unwrap().write(buf) {
+ Ok(n) => {
+ // if we write only part of our buffer, this is sufficient on unix to show
+ // that the socket buffer is full
+ if n > 0 && (!cfg!(windows) && n < buf.len()) {
+ self.registration.clear_readiness(evt);
+ }
+
+ return Poll::Ready(Ok(n));
+ },
+ Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+ self.registration.clear_readiness(evt);
+ }
+ Err(e) => return Poll::Ready(Err(e)),
+ }
+ }
}
- #[cfg(feature = "net")]
+ #[cfg(any(feature = "net", feature = "process"))]
pub(crate) fn poll_write_vectored<'a>(
&'a self,
cx: &mut Context<'_>,
@@ -185,6 +223,10 @@
}
}
+impl<E: Source> UnwindSafe for PollEvented<E> {}
+
+impl<E: Source> RefUnwindSafe for PollEvented<E> {}
+
impl<E: Source> Deref for PollEvented<E> {
type Target = E;
diff --git a/src/io/read_buf.rs b/src/io/read_buf.rs
index ad58cbe..0dc595a 100644
--- a/src/io/read_buf.rs
+++ b/src/io/read_buf.rs
@@ -1,9 +1,5 @@
-// This lint claims ugly casting is somehow safer than transmute, but there's
-// no evidence that is the case. Shush.
-#![allow(clippy::transmute_ptr_to_ptr)]
-
use std::fmt;
-use std::mem::{self, MaybeUninit};
+use std::mem::MaybeUninit;
/// A wrapper around a byte buffer that is incrementally filled and initialized.
///
@@ -35,7 +31,7 @@
#[inline]
pub fn new(buf: &'a mut [u8]) -> ReadBuf<'a> {
let initialized = buf.len();
- let buf = unsafe { mem::transmute::<&mut [u8], &mut [MaybeUninit<u8>]>(buf) };
+ let buf = unsafe { slice_to_uninit_mut(buf) };
ReadBuf {
buf,
filled: 0,
@@ -67,8 +63,7 @@
let slice = &self.buf[..self.filled];
// safety: filled describes how far into the buffer that the
// user has filled with bytes, so it's been initialized.
- // TODO: This could use `MaybeUninit::slice_get_ref` when it is stable.
- unsafe { mem::transmute::<&[MaybeUninit<u8>], &[u8]>(slice) }
+ unsafe { slice_assume_init(slice) }
}
/// Returns a mutable reference to the filled portion of the buffer.
@@ -77,8 +72,7 @@
let slice = &mut self.buf[..self.filled];
// safety: filled describes how far into the buffer that the
// user has filled with bytes, so it's been initialized.
- // TODO: This could use `MaybeUninit::slice_get_mut` when it is stable.
- unsafe { mem::transmute::<&mut [MaybeUninit<u8>], &mut [u8]>(slice) }
+ unsafe { slice_assume_init_mut(slice) }
}
/// Returns a new `ReadBuf` comprised of the unfilled section up to `n`.
@@ -97,8 +91,7 @@
let slice = &self.buf[..self.initialized];
// safety: initialized describes how far into the buffer that the
// user has at some point initialized with bytes.
- // TODO: This could use `MaybeUninit::slice_get_ref` when it is stable.
- unsafe { mem::transmute::<&[MaybeUninit<u8>], &[u8]>(slice) }
+ unsafe { slice_assume_init(slice) }
}
/// Returns a mutable reference to the initialized portion of the buffer.
@@ -109,15 +102,14 @@
let slice = &mut self.buf[..self.initialized];
// safety: initialized describes how far into the buffer that the
// user has at some point initialized with bytes.
- // TODO: This could use `MaybeUninit::slice_get_mut` when it is stable.
- unsafe { mem::transmute::<&mut [MaybeUninit<u8>], &mut [u8]>(slice) }
+ unsafe { slice_assume_init_mut(slice) }
}
/// Returns a mutable reference to the entire buffer, without ensuring that it has been fully
/// initialized.
///
/// The elements between 0 and `self.filled().len()` are filled, and those between 0 and
- /// `self.initialized().len()` are initialized (and so can be transmuted to a `&mut [u8]`).
+ /// `self.initialized().len()` are initialized (and so can be converted to a `&mut [u8]`).
///
/// The caller of this method must ensure that these invariants are upheld. For example, if the
/// caller initializes some of the uninitialized section of the buffer, it must call
@@ -160,6 +152,7 @@
///
/// Panics if `self.remaining()` is less than `n`.
#[inline]
+ #[track_caller]
pub fn initialize_unfilled_to(&mut self, n: usize) -> &mut [u8] {
assert!(self.remaining() >= n, "n overflows remaining");
@@ -178,7 +171,7 @@
let slice = &mut self.buf[self.filled..end];
// safety: just above, we checked that the end of the buf has
// been initialized to some value.
- unsafe { mem::transmute::<&mut [MaybeUninit<u8>], &mut [u8]>(slice) }
+ unsafe { slice_assume_init_mut(slice) }
}
/// Returns the number of bytes at the end of the slice that have not yet been filled.
@@ -203,6 +196,7 @@
///
/// Panics if the filled region of the buffer would become larger than the initialized region.
#[inline]
+ #[track_caller]
pub fn advance(&mut self, n: usize) {
let new = self.filled.checked_add(n).expect("filled overflow");
self.set_filled(new);
@@ -219,6 +213,7 @@
///
/// Panics if the filled region of the buffer would become larger than the initialized region.
#[inline]
+ #[track_caller]
pub fn set_filled(&mut self, n: usize) {
assert!(
n <= self.initialized,
@@ -249,6 +244,7 @@
///
/// Panics if `self.remaining()` is less than `buf.len()`.
#[inline]
+ #[track_caller]
pub fn put_slice(&mut self, buf: &[u8]) {
assert!(
self.remaining() >= buf.len(),
@@ -283,3 +279,17 @@
.finish()
}
}
+
+unsafe fn slice_to_uninit_mut(slice: &mut [u8]) -> &mut [MaybeUninit<u8>] {
+ &mut *(slice as *mut [u8] as *mut [MaybeUninit<u8>])
+}
+
+// TODO: This could use `MaybeUninit::slice_assume_init` when it is stable.
+unsafe fn slice_assume_init(slice: &[MaybeUninit<u8>]) -> &[u8] {
+ &*(slice as *const [MaybeUninit<u8>] as *const [u8])
+}
+
+// TODO: This could use `MaybeUninit::slice_assume_init_mut` when it is stable.
+unsafe fn slice_assume_init_mut(slice: &mut [MaybeUninit<u8>]) -> &mut [u8] {
+ &mut *(slice as *mut [MaybeUninit<u8>] as *mut [u8])
+}
diff --git a/src/io/driver/ready.rs b/src/io/ready.rs
similarity index 98%
rename from src/io/driver/ready.rs
rename to src/io/ready.rs
index 2430d30..ef135c4 100644
--- a/src/io/driver/ready.rs
+++ b/src/io/ready.rs
@@ -12,7 +12,7 @@
///
/// `Ready` tracks which operation an I/O resource is ready to perform.
#[cfg_attr(docsrs, doc(cfg(feature = "net")))]
-#[derive(Clone, Copy, PartialEq, PartialOrd)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Ready(usize);
impl Ready {
diff --git a/src/io/split.rs b/src/io/split.rs
index 8258a0f..f067b65 100644
--- a/src/io/split.rs
+++ b/src/io/split.rs
@@ -74,7 +74,11 @@
/// same `split` operation this method will panic.
/// This can be checked ahead of time by comparing the stream ID
/// of the two halves.
- pub fn unsplit(self, wr: WriteHalf<T>) -> T {
+ #[track_caller]
+ pub fn unsplit(self, wr: WriteHalf<T>) -> T
+ where
+ T: Unpin,
+ {
if self.is_pair_of(&wr) {
drop(wr);
diff --git a/src/io/stdio_common.rs b/src/io/stdio_common.rs
index 7e4a198..b1cc61d 100644
--- a/src/io/stdio_common.rs
+++ b/src/io/stdio_common.rs
@@ -42,7 +42,7 @@
// for further code. Since `AsyncWrite` can always shrink
// buffer at its discretion, excessive (i.e. in tests) shrinking
// does not break correctness.
- // 2. If buffer is small, it will not be shrinked.
+ // 2. If buffer is small, it will not be shrunk.
// That's why, it's "textness" will not change, so we don't have
// to fixup it.
if cfg!(not(any(target_os = "windows", test))) || buf.len() <= crate::io::blocking::MAX_BUF
@@ -108,14 +108,13 @@
#[cfg(test)]
#[cfg(not(loom))]
mod tests {
+ use crate::io::blocking::MAX_BUF;
use crate::io::AsyncWriteExt;
use std::io;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
- const MAX_BUF: usize = 16 * 1024;
-
struct TextMockWriter;
impl crate::io::AsyncWrite for TextMockWriter {
@@ -193,7 +192,7 @@
fn test_pseudo_text() {
// In this test we write a piece of binary data, whose beginning is
// text though. We then validate that even in this corner case buffer
- // was not shrinked too much.
+ // was not shrunk too much.
let checked_count = super::MAGIC_CONST * super::MAX_BYTES_PER_CHAR;
let mut data: Vec<u8> = str::repeat("a", checked_count).into();
data.extend(std::iter::repeat(0b1010_1010).take(MAX_BUF - checked_count + 1));
@@ -212,7 +211,7 @@
writer.write_history.iter().copied().sum::<usize>(),
data.len()
);
- // Check that at most MAX_BYTES_PER_CHAR + 1 (i.e. 5) bytes were shrinked
+ // Check that at most MAX_BYTES_PER_CHAR + 1 (i.e. 5) bytes were shrunk
// from the buffer: one because it was outside of MAX_BUF boundary, and
// up to one "utf8 code point".
assert!(data.len() - writer.write_history[0] <= super::MAX_BYTES_PER_CHAR + 1);
diff --git a/src/io/util/async_seek_ext.rs b/src/io/util/async_seek_ext.rs
index 46b3e6c..aadf3a7 100644
--- a/src/io/util/async_seek_ext.rs
+++ b/src/io/util/async_seek_ext.rs
@@ -69,7 +69,7 @@
/// Creates a future which will rewind to the beginning of the stream.
///
- /// This is convenience method, equivalent to to `self.seek(SeekFrom::Start(0))`.
+ /// This is convenience method, equivalent to `self.seek(SeekFrom::Start(0))`.
fn rewind(&mut self) -> Seek<'_, Self>
where
Self: Unpin,
diff --git a/src/io/util/async_write_ext.rs b/src/io/util/async_write_ext.rs
index 93a3183..dfdde82 100644
--- a/src/io/util/async_write_ext.rs
+++ b/src/io/util/async_write_ext.rs
@@ -406,7 +406,7 @@
/// ```
fn write_u8(&mut self, n: u8) -> WriteU8;
- /// Writes an unsigned 8-bit integer to the underlying writer.
+ /// Writes a signed 8-bit integer to the underlying writer.
///
/// Equivalent to:
///
@@ -425,7 +425,7 @@
///
/// # Examples
///
- /// Write unsigned 8 bit integers to a `AsyncWrite`:
+ /// Write signed 8 bit integers to a `AsyncWrite`:
///
/// ```rust
/// use tokio::io::{self, AsyncWriteExt};
@@ -434,10 +434,10 @@
/// async fn main() -> io::Result<()> {
/// let mut writer = Vec::new();
///
- /// writer.write_u8(2).await?;
- /// writer.write_u8(5).await?;
+ /// writer.write_i8(-2).await?;
+ /// writer.write_i8(126).await?;
///
- /// assert_eq!(writer, b"\x02\x05");
+ /// assert_eq!(writer, b"\xFE\x7E");
/// Ok(())
/// }
/// ```
diff --git a/src/io/util/buf_reader.rs b/src/io/util/buf_reader.rs
index 7df610b..60879c0 100644
--- a/src/io/util/buf_reader.rs
+++ b/src/io/util/buf_reader.rs
@@ -204,7 +204,6 @@
self.as_mut()
.get_pin_mut()
.start_seek(SeekFrom::Current(offset))?;
- self.as_mut().get_pin_mut().poll_complete(cx)?
} else {
// seek backwards by our remainder, and then by the offset
self.as_mut()
@@ -221,8 +220,8 @@
self.as_mut()
.get_pin_mut()
.start_seek(SeekFrom::Current(n))?;
- self.as_mut().get_pin_mut().poll_complete(cx)?
}
+ self.as_mut().get_pin_mut().poll_complete(cx)?
}
SeekState::PendingOverflowed(n) => {
if self.as_mut().get_pin_mut().poll_complete(cx)?.is_pending() {
diff --git a/src/io/util/copy.rs b/src/io/util/copy.rs
index d0ab7cb..47dad89 100644
--- a/src/io/util/copy.rs
+++ b/src/io/util/copy.rs
@@ -27,6 +27,51 @@
}
}
+ fn poll_fill_buf<R>(
+ &mut self,
+ cx: &mut Context<'_>,
+ reader: Pin<&mut R>,
+ ) -> Poll<io::Result<()>>
+ where
+ R: AsyncRead + ?Sized,
+ {
+ let me = &mut *self;
+ let mut buf = ReadBuf::new(&mut me.buf);
+ buf.set_filled(me.cap);
+
+ let res = reader.poll_read(cx, &mut buf);
+ if let Poll::Ready(Ok(_)) = res {
+ let filled_len = buf.filled().len();
+ me.read_done = me.cap == filled_len;
+ me.cap = filled_len;
+ }
+ res
+ }
+
+ fn poll_write_buf<R, W>(
+ &mut self,
+ cx: &mut Context<'_>,
+ mut reader: Pin<&mut R>,
+ mut writer: Pin<&mut W>,
+ ) -> Poll<io::Result<usize>>
+ where
+ R: AsyncRead + ?Sized,
+ W: AsyncWrite + ?Sized,
+ {
+ let me = &mut *self;
+ match writer.as_mut().poll_write(cx, &me.buf[me.pos..me.cap]) {
+ Poll::Pending => {
+ // Top up the buffer towards full if we can read a bit more
+ // data - this should improve the chances of a large write
+ if !me.read_done && me.cap < me.buf.len() {
+ ready!(me.poll_fill_buf(cx, reader.as_mut()))?;
+ }
+ Poll::Pending
+ }
+ res => res,
+ }
+ }
+
pub(super) fn poll_copy<R, W>(
&mut self,
cx: &mut Context<'_>,
@@ -41,10 +86,10 @@
// If our buffer is empty, then we need to read some data to
// continue.
if self.pos == self.cap && !self.read_done {
- let me = &mut *self;
- let mut buf = ReadBuf::new(&mut me.buf);
+ self.pos = 0;
+ self.cap = 0;
- match reader.as_mut().poll_read(cx, &mut buf) {
+ match self.poll_fill_buf(cx, reader.as_mut()) {
Poll::Ready(Ok(_)) => (),
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
Poll::Pending => {
@@ -58,20 +103,11 @@
return Poll::Pending;
}
}
-
- let n = buf.filled().len();
- if n == 0 {
- self.read_done = true;
- } else {
- self.pos = 0;
- self.cap = n;
- }
}
// If our buffer has some data, let's write it out!
while self.pos < self.cap {
- let me = &mut *self;
- let i = ready!(writer.as_mut().poll_write(cx, &me.buf[me.pos..me.cap]))?;
+ let i = ready!(self.poll_write_buf(cx, reader.as_mut(), writer.as_mut()))?;
if i == 0 {
return Poll::Ready(Err(io::Error::new(
io::ErrorKind::WriteZero,
diff --git a/src/io/util/empty.rs b/src/io/util/empty.rs
index f964d18..9e648f8 100644
--- a/src/io/util/empty.rs
+++ b/src/io/util/empty.rs
@@ -50,16 +50,18 @@
#[inline]
fn poll_read(
self: Pin<&mut Self>,
- _: &mut Context<'_>,
+ cx: &mut Context<'_>,
_: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
+ ready!(poll_proceed_and_make_progress(cx));
Poll::Ready(Ok(()))
}
}
impl AsyncBufRead for Empty {
#[inline]
- fn poll_fill_buf(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
+ fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
+ ready!(poll_proceed_and_make_progress(cx));
Poll::Ready(Ok(&[]))
}
@@ -73,6 +75,20 @@
}
}
+cfg_coop! {
+ fn poll_proceed_and_make_progress(cx: &mut Context<'_>) -> Poll<()> {
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
+ coop.made_progress();
+ Poll::Ready(())
+ }
+}
+
+cfg_not_coop! {
+ fn poll_proceed_and_make_progress(_: &mut Context<'_>) -> Poll<()> {
+ Poll::Ready(())
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/src/io/util/fill_buf.rs b/src/io/util/fill_buf.rs
index 3655c01..bb07c76 100644
--- a/src/io/util/fill_buf.rs
+++ b/src/io/util/fill_buf.rs
@@ -40,6 +40,12 @@
// Safety: This is necessary only due to a limitation in the
// borrow checker. Once Rust starts using the polonius borrow
// checker, this can be simplified.
+ //
+ // The safety of this transmute relies on the fact that the
+ // value of `reader` is `None` when we return in this branch.
+ // Otherwise the caller could poll us again after
+ // completion, and access the mutable reference while the
+ // returned immutable reference still exists.
let slice = std::mem::transmute::<&[u8], &'a [u8]>(slice);
Poll::Ready(Ok(slice))
},
diff --git a/src/io/util/mem.rs b/src/io/util/mem.rs
index 4eefe7b..31884b3 100644
--- a/src/io/util/mem.rs
+++ b/src/io/util/mem.rs
@@ -177,10 +177,8 @@
waker.wake();
}
}
-}
-impl AsyncRead for Pipe {
- fn poll_read(
+ fn poll_read_internal(
mut self: Pin<&mut Self>,
cx: &mut task::Context<'_>,
buf: &mut ReadBuf<'_>,
@@ -204,10 +202,8 @@
Poll::Pending
}
}
-}
-impl AsyncWrite for Pipe {
- fn poll_write(
+ fn poll_write_internal(
mut self: Pin<&mut Self>,
cx: &mut task::Context<'_>,
buf: &[u8],
@@ -228,6 +224,62 @@
}
Poll::Ready(Ok(len))
}
+}
+
+impl AsyncRead for Pipe {
+ cfg_coop! {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut task::Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<std::io::Result<()>> {
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
+
+ let ret = self.poll_read_internal(cx, buf);
+ if ret.is_ready() {
+ coop.made_progress();
+ }
+ ret
+ }
+ }
+
+ cfg_not_coop! {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut task::Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<std::io::Result<()>> {
+ self.poll_read_internal(cx, buf)
+ }
+ }
+}
+
+impl AsyncWrite for Pipe {
+ cfg_coop! {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ cx: &mut task::Context<'_>,
+ buf: &[u8],
+ ) -> Poll<std::io::Result<usize>> {
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
+
+ let ret = self.poll_write_internal(cx, buf);
+ if ret.is_ready() {
+ coop.made_progress();
+ }
+ ret
+ }
+ }
+
+ cfg_not_coop! {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ cx: &mut task::Context<'_>,
+ buf: &[u8],
+ ) -> Poll<std::io::Result<usize>> {
+ self.poll_write_internal(cx, buf)
+ }
+ }
fn poll_flush(self: Pin<&mut Self>, _: &mut task::Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
diff --git a/src/io/util/read.rs b/src/io/util/read.rs
index edc9d5a..a1f9c8a 100644
--- a/src/io/util/read.rs
+++ b/src/io/util/read.rs
@@ -48,7 +48,7 @@
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
let me = self.project();
- let mut buf = ReadBuf::new(*me.buf);
+ let mut buf = ReadBuf::new(me.buf);
ready!(Pin::new(me.reader).poll_read(cx, &mut buf))?;
Poll::Ready(Ok(buf.filled().len()))
}
diff --git a/src/io/util/read_exact.rs b/src/io/util/read_exact.rs
index 1e8150e..dbdd58b 100644
--- a/src/io/util/read_exact.rs
+++ b/src/io/util/read_exact.rs
@@ -51,13 +51,13 @@
type Output = io::Result<usize>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
- let mut me = self.project();
+ let me = self.project();
loop {
// if our buffer is empty, then we need to read some data to continue.
let rem = me.buf.remaining();
if rem != 0 {
- ready!(Pin::new(&mut *me.reader).poll_read(cx, &mut me.buf))?;
+ ready!(Pin::new(&mut *me.reader).poll_read(cx, me.buf))?;
if me.buf.remaining() == rem {
return Err(eof()).into();
}
diff --git a/src/io/util/take.rs b/src/io/util/take.rs
index b5e90c9..df2f61b 100644
--- a/src/io/util/take.rs
+++ b/src/io/util/take.rs
@@ -86,7 +86,11 @@
let me = self.project();
let mut b = buf.take(*me.limit_ as usize);
+
+ let buf_ptr = b.filled().as_ptr();
ready!(me.inner.poll_read(cx, &mut b))?;
+ assert_eq!(b.filled().as_ptr(), buf_ptr);
+
let n = b.filled().len();
// We need to update the original ReadBuf
diff --git a/src/io/util/vec_with_initialized.rs b/src/io/util/vec_with_initialized.rs
index 208cc93..a9b94e3 100644
--- a/src/io/util/vec_with_initialized.rs
+++ b/src/io/util/vec_with_initialized.rs
@@ -1,19 +1,18 @@
use crate::io::ReadBuf;
use std::mem::MaybeUninit;
-mod private {
- pub trait Sealed {}
+/// Something that looks like a `Vec<u8>`.
+///
+/// # Safety
+///
+/// The implementor must guarantee that the vector returned by the
+/// `as_mut` and `as_mut` methods do not change from one call to
+/// another.
+pub(crate) unsafe trait VecU8: AsRef<Vec<u8>> + AsMut<Vec<u8>> {}
- impl Sealed for Vec<u8> {}
- impl Sealed for &mut Vec<u8> {}
-}
+unsafe impl VecU8 for Vec<u8> {}
+unsafe impl VecU8 for &mut Vec<u8> {}
-/// A sealed trait that constrains the generic type parameter in `VecWithInitialized<V>`. That struct's safety relies
-/// on certain invariants upheld by `Vec<u8>`.
-pub(crate) trait VecU8: AsMut<Vec<u8>> + private::Sealed {}
-
-impl VecU8 for Vec<u8> {}
-impl VecU8 for &mut Vec<u8> {}
/// This struct wraps a `Vec<u8>` or `&mut Vec<u8>`, combining it with a
/// `num_initialized`, which keeps track of the number of initialized bytes
/// in the unused capacity.
@@ -64,8 +63,8 @@
}
#[cfg(feature = "io-util")]
- pub(crate) fn is_empty(&mut self) -> bool {
- self.vec.as_mut().is_empty()
+ pub(crate) fn is_empty(&self) -> bool {
+ self.vec.as_ref().is_empty()
}
pub(crate) fn get_read_buf<'a>(&'a mut self) -> ReadBuf<'a> {
diff --git a/src/io/util/write_all.rs b/src/io/util/write_all.rs
index e59d41e..abd3e39 100644
--- a/src/io/util/write_all.rs
+++ b/src/io/util/write_all.rs
@@ -42,7 +42,7 @@
while !me.buf.is_empty() {
let n = ready!(Pin::new(&mut *me.writer).poll_write(cx, me.buf))?;
{
- let (_, rest) = mem::replace(&mut *me.buf, &[]).split_at(n);
+ let (_, rest) = mem::take(&mut *me.buf).split_at(n);
*me.buf = rest;
}
if n == 0 {
diff --git a/src/lib.rs b/src/lib.rs
index 9821c1a..05767d0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,7 +1,9 @@
#![allow(
clippy::cognitive_complexity,
clippy::large_enum_variant,
- clippy::needless_doctest_main
+ clippy::module_inception,
+ clippy::needless_doctest_main,
+ clippy::declare_interior_mutable_const
)]
#![warn(
missing_debug_implementations,
@@ -10,17 +12,13 @@
unreachable_pub
)]
#![deny(unused_must_use)]
-#![cfg_attr(docsrs, deny(rustdoc::broken_intra_doc_links))]
#![doc(test(
no_crate_inject,
attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables))
))]
#![cfg_attr(docsrs, feature(doc_cfg))]
-#![cfg_attr(docsrs, feature(doc_cfg_hide))]
-#![cfg_attr(docsrs, doc(cfg_hide(docsrs)))]
-#![cfg_attr(docsrs, doc(cfg_hide(loom)))]
-#![cfg_attr(docsrs, doc(cfg_hide(not(loom))))]
#![cfg_attr(docsrs, allow(unused_attributes))]
+#![cfg_attr(loom, allow(dead_code, unreachable_pub))]
//! A runtime for writing reliable network applications without compromising speed.
//!
@@ -119,7 +117,7 @@
//! The [`tokio::sync`] module contains synchronization primitives to use when
//! needing to communicate or share data. These include:
//!
-//! * channels ([`oneshot`], [`mpsc`], and [`watch`]), for sending values
+//! * channels ([`oneshot`], [`mpsc`], [`watch`], and [`broadcast`]), for sending values
//! between tasks,
//! * a non-blocking [`Mutex`], for controlling access to a shared, mutable
//! value,
@@ -135,6 +133,7 @@
//! [`oneshot`]: crate::sync::oneshot
//! [`mpsc`]: crate::sync::mpsc
//! [`watch`]: crate::sync::watch
+//! [`broadcast`]: crate::sync::broadcast
//!
//! The [`tokio::time`] module provides utilities for tracking time and
//! scheduling work. This includes functions for setting [timeouts][timeout] for
@@ -156,7 +155,7 @@
//! provide the functionality you need.
//!
//! Using the runtime requires the "rt" or "rt-multi-thread" feature flags, to
-//! enable the basic [single-threaded scheduler][rt] and the [thread-pool
+//! enable the current-thread [single-threaded scheduler][rt] and the [multi-thread
//! scheduler][rt-multi-thread], respectively. See the [`runtime` module
//! documentation][rt-features] for details. In addition, the "macros" feature
//! flag enables the `#[tokio::main]` and `#[tokio::test]` attributes.
@@ -175,12 +174,15 @@
//! swapping the currently running task on each thread. However, this kind of
//! swapping can only happen at `.await` points, so code that spends a long time
//! without reaching an `.await` will prevent other tasks from running. To
-//! combat this, Tokio provides two kinds of threads: Core threads and blocking
-//! threads. The core threads are where all asynchronous code runs, and Tokio
-//! will by default spawn one for each CPU core. The blocking threads are
-//! spawned on demand, can be used to run blocking code that would otherwise
-//! block other tasks from running and are kept alive when not used for a certain
-//! amount of time which can be configured with [`thread_keep_alive`].
+//! combat this, Tokio provides two kinds of threads: Core threads and blocking threads.
+//!
+//! The core threads are where all asynchronous code runs, and Tokio will by default
+//! spawn one for each CPU core. You can use the environment variable `TOKIO_WORKER_THREADS`
+//! to override the default value.
+//!
+//! The blocking threads are spawned on demand, can be used to run blocking code
+//! that would otherwise block other tasks from running and are kept alive when
+//! not used for a certain amount of time which can be configured with [`thread_keep_alive`].
//! Since it is not possible for Tokio to swap out blocking tasks, like it
//! can do with asynchronous code, the upper limit on the number of blocking
//! threads is very large. These limits can be configured on the [`Builder`].
@@ -312,8 +314,8 @@
//! Beware though that this will pull in many extra dependencies that you may not
//! need.
//!
-//! - `full`: Enables all Tokio public API features listed below except `test-util`.
-//! - `rt`: Enables `tokio::spawn`, the basic (current thread) scheduler,
+//! - `full`: Enables all features listed below except `test-util` and `tracing`.
+//! - `rt`: Enables `tokio::spawn`, the current-thread scheduler,
//! and non-scheduler utilities.
//! - `rt-multi-thread`: Enables the heavier, multi-threaded, work-stealing scheduler.
//! - `io-util`: Enables the IO based `Ext` traits.
@@ -329,30 +331,91 @@
//! - `signal`: Enables all `tokio::signal` types.
//! - `fs`: Enables `tokio::fs` types.
//! - `test-util`: Enables testing based infrastructure for the Tokio runtime.
+//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's
+//! synchronization primitives internally. Also, this
+//! dependency is necessary to construct some of our primitives
+//! in a const context. MSRV may increase according to the
+//! _parking_lot_ release in use.
//!
//! _Note: `AsyncRead` and `AsyncWrite` traits do not require any features and are
//! always available._
//!
-//! ### Internal features
-//!
-//! These features do not expose any new API, but influence internal
-//! implementation aspects of Tokio, and can pull in additional
-//! dependencies.
-//!
-//! - `parking_lot`: As a potential optimization, use the _parking_lot_ crate's
-//! synchronization primitives internally. MSRV may increase according to the
-//! _parking_lot_ release in use.
-//!
//! ### Unstable features
//!
-//! These feature flags enable **unstable** features. The public API may break in 1.x
-//! releases. To enable these features, the `--cfg tokio_unstable` must be passed to
-//! `rustc` when compiling. This is easiest done using the `RUSTFLAGS` env variable:
-//! `RUSTFLAGS="--cfg tokio_unstable"`.
+//! Some feature flags are only available when specifying the `tokio_unstable` flag:
//!
//! - `tracing`: Enables tracing events.
//!
+//! Likewise, some parts of the API are only available with the same flag:
+//!
+//! - [`task::Builder`]
+//! - Some methods on [`task::JoinSet`]
+//! - [`runtime::RuntimeMetrics`]
+//! - [`runtime::Builder::unhandled_panic`]
+//! - [`task::Id`]
+//!
+//! This flag enables **unstable** features. The public API of these features
+//! may break in 1.x releases. To enable these features, the `--cfg
+//! tokio_unstable` argument must be passed to `rustc` when compiling. This
+//! serves to explicitly opt-in to features which may break semver conventions,
+//! since Cargo [does not yet directly support such opt-ins][unstable features].
+//!
+//! You can specify it in your project's `.cargo/config.toml` file:
+//!
+//! ```toml
+//! [build]
+//! rustflags = ["--cfg", "tokio_unstable"]
+//! ```
+//!
+//! Alternatively, you can specify it with an environment variable:
+//!
+//! ```sh
+//! ## Many *nix shells:
+//! export RUSTFLAGS="--cfg tokio_unstable"
+//! cargo build
+//! ```
+//!
+//! ```powershell
+//! ## Windows PowerShell:
+//! $Env:RUSTFLAGS="--cfg tokio_unstable"
+//! cargo build
+//! ```
+//!
+//! [unstable features]: https://internals.rust-lang.org/t/feature-request-unstable-opt-in-non-transitive-crate-features/16193#why-not-a-crate-feature-2
//! [feature flags]: https://doc.rust-lang.org/cargo/reference/manifest.html#the-features-section
+//!
+//! ## WASM support
+//!
+//! Tokio has some limited support for the WASM platform. Without the
+//! `tokio_unstable` flag, the following features are supported:
+//!
+//! * `sync`
+//! * `macros`
+//! * `io-util`
+//! * `rt`
+//! * `time`
+//!
+//! Enabling any other feature (including `full`) will cause a compilation
+//! failure.
+//!
+//! The `time` module will only work on WASM platforms that have support for
+//! timers (e.g. wasm32-wasi). The timing functions will panic if used on a WASM
+//! platform that does not support timers.
+//!
+//! Note also that if the runtime becomes indefinitely idle, it will panic
+//! immediately instead of blocking forever. On platforms that don't support
+//! time, this means that the runtime can never be idle in any way.
+//!
+//! ### Unstable WASM support
+//!
+//! Tokio also has unstable support for some additional WASM features. This
+//! requires the use of the `tokio_unstable` flag.
+//!
+//! Using this flag enables the use of `tokio::net` on the wasm32-wasi target.
+//! However, not all methods are available on the networking types as WASI
+//! currently does not support the creation of new sockets from within WASM.
+//! Because of this, sockets must currently be created via the `FromRawFd`
+//! trait.
// Test that pointer width is compatible. This asserts that e.g. usize is at
// least 32 bits, which a lot of components in Tokio currently assumes.
@@ -367,6 +430,37 @@
"Tokio requires the platform pointer width to be 32, 64, or 128 bits"
}
+// Ensure that our build script has correctly set cfg flags for wasm.
+//
+// Each condition is written all(a, not(b)). This should be read as
+// "if a, then we must also have b".
+#[cfg(any(
+ all(target_arch = "wasm32", not(tokio_wasm)),
+ all(target_arch = "wasm64", not(tokio_wasm)),
+ all(target_family = "wasm", not(tokio_wasm)),
+ all(target_os = "wasi", not(tokio_wasm)),
+ all(target_os = "wasi", not(tokio_wasi)),
+ all(target_os = "wasi", tokio_wasm_not_wasi),
+ all(tokio_wasm, not(any(target_arch = "wasm32", target_arch = "wasm64"))),
+ all(tokio_wasm_not_wasi, not(tokio_wasm)),
+ all(tokio_wasi, not(tokio_wasm))
+))]
+compile_error!("Tokio's build script has incorrectly detected wasm.");
+
+#[cfg(all(
+ not(tokio_unstable),
+ tokio_wasm,
+ any(
+ feature = "fs",
+ feature = "io-std",
+ feature = "net",
+ feature = "process",
+ feature = "rt-multi-thread",
+ feature = "signal"
+ )
+))]
+compile_error!("Only features sync,macros,io-util,rt,time are supported on wasm.");
+
// Includes re-exports used by macros.
//
// This module is not intended to be part of the public API. In general, any
@@ -385,20 +479,25 @@
pub mod net;
mod loom;
-mod park;
cfg_process! {
pub mod process;
}
-#[cfg(any(feature = "net", feature = "fs", feature = "io-std"))]
+#[cfg(any(
+ feature = "fs",
+ feature = "io-std",
+ feature = "net",
+ all(windows, feature = "process"),
+))]
mod blocking;
cfg_rt! {
pub mod runtime;
}
-
-pub(crate) mod coop;
+cfg_not_rt! {
+ pub(crate) mod runtime;
+}
cfg_signal! {
pub mod signal;
@@ -482,14 +581,6 @@
#[allow(unused)]
pub(crate) use std::os;
-#[cfg(docsrs)]
-#[allow(unused)]
-pub(crate) use self::doc::winapi;
-
-#[cfg(all(not(docsrs), windows, feature = "net"))]
-#[allow(unused)]
-pub(crate) use ::winapi;
-
cfg_macros! {
/// Implementation detail of the `select!` macro. This macro is **not**
/// intended to be used as part of the public API and is permitted to
diff --git a/src/loom/mocked.rs b/src/loom/mocked.rs
index 367d59b..56dc1a0 100644
--- a/src/loom/mocked.rs
+++ b/src/loom/mocked.rs
@@ -25,6 +25,13 @@
}
}
pub(crate) use loom::sync::*;
+
+ pub(crate) mod atomic {
+ pub(crate) use loom::sync::atomic::*;
+
+ // TODO: implement a loom version
+ pub(crate) type StaticAtomicU64 = std::sync::atomic::AtomicU64;
+ }
}
pub(crate) mod rand {
@@ -38,3 +45,8 @@
2
}
}
+
+pub(crate) mod thread {
+ pub use loom::lazy_static::AccessError;
+ pub use loom::thread::*;
+}
diff --git a/src/loom/std/atomic_ptr.rs b/src/loom/std/atomic_ptr.rs
deleted file mode 100644
index 236645f..0000000
--- a/src/loom/std/atomic_ptr.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use std::fmt;
-use std::ops::{Deref, DerefMut};
-
-/// `AtomicPtr` providing an additional `load_unsync` function.
-pub(crate) struct AtomicPtr<T> {
- inner: std::sync::atomic::AtomicPtr<T>,
-}
-
-impl<T> AtomicPtr<T> {
- pub(crate) fn new(ptr: *mut T) -> AtomicPtr<T> {
- let inner = std::sync::atomic::AtomicPtr::new(ptr);
- AtomicPtr { inner }
- }
-}
-
-impl<T> Deref for AtomicPtr<T> {
- type Target = std::sync::atomic::AtomicPtr<T>;
-
- fn deref(&self) -> &Self::Target {
- &self.inner
- }
-}
-
-impl<T> DerefMut for AtomicPtr<T> {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.inner
- }
-}
-
-impl<T> fmt::Debug for AtomicPtr<T> {
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- self.deref().fmt(fmt)
- }
-}
diff --git a/src/loom/std/atomic_u16.rs b/src/loom/std/atomic_u16.rs
index c1c5312..c9e105c 100644
--- a/src/loom/std/atomic_u16.rs
+++ b/src/loom/std/atomic_u16.rs
@@ -2,7 +2,7 @@
use std::fmt;
use std::ops::Deref;
-/// `AtomicU16` providing an additional `load_unsync` function.
+/// `AtomicU16` providing an additional `unsync_load` function.
pub(crate) struct AtomicU16 {
inner: UnsafeCell<std::sync::atomic::AtomicU16>,
}
@@ -23,7 +23,7 @@
/// All mutations must have happened before the unsynchronized load.
/// Additionally, there must be no concurrent mutations.
pub(crate) unsafe fn unsync_load(&self) -> u16 {
- *(*self.inner.get()).get_mut()
+ core::ptr::read(self.inner.get() as *const u16)
}
}
diff --git a/src/loom/std/atomic_u32.rs b/src/loom/std/atomic_u32.rs
index 61f95fb..ee0d2d3 100644
--- a/src/loom/std/atomic_u32.rs
+++ b/src/loom/std/atomic_u32.rs
@@ -2,7 +2,7 @@
use std::fmt;
use std::ops::Deref;
-/// `AtomicU32` providing an additional `load_unsync` function.
+/// `AtomicU32` providing an additional `unsync_load` function.
pub(crate) struct AtomicU32 {
inner: UnsafeCell<std::sync::atomic::AtomicU32>,
}
@@ -15,6 +15,16 @@
let inner = UnsafeCell::new(std::sync::atomic::AtomicU32::new(val));
AtomicU32 { inner }
}
+
+ /// Performs an unsynchronized load.
+ ///
+ /// # Safety
+ ///
+ /// All mutations must have happened before the unsynchronized load.
+ /// Additionally, there must be no concurrent mutations.
+ pub(crate) unsafe fn unsync_load(&self) -> u32 {
+ core::ptr::read(self.inner.get() as *const u32)
+ }
}
impl Deref for AtomicU32 {
diff --git a/src/loom/std/atomic_u64.rs b/src/loom/std/atomic_u64.rs
index 8ea6bd4..ce391be 100644
--- a/src/loom/std/atomic_u64.rs
+++ b/src/loom/std/atomic_u64.rs
@@ -7,65 +7,13 @@
// `#[cfg(target_has_atomic = "64")]`.
// Refs: https://github.com/rust-lang/rust/tree/master/src/librustc_target
cfg_has_atomic_u64! {
- pub(crate) use std::sync::atomic::AtomicU64;
+ #[path = "atomic_u64_native.rs"]
+ mod imp;
}
cfg_not_has_atomic_u64! {
- use crate::loom::sync::Mutex;
- use std::sync::atomic::Ordering;
-
- #[derive(Debug)]
- pub(crate) struct AtomicU64 {
- inner: Mutex<u64>,
- }
-
- impl AtomicU64 {
- pub(crate) fn new(val: u64) -> Self {
- Self {
- inner: Mutex::new(val),
- }
- }
-
- pub(crate) fn load(&self, _: Ordering) -> u64 {
- *self.inner.lock()
- }
-
- pub(crate) fn store(&self, val: u64, _: Ordering) {
- *self.inner.lock() = val;
- }
-
- pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
- let mut lock = self.inner.lock();
- let prev = *lock;
- *lock = prev | val;
- prev
- }
-
- pub(crate) fn compare_exchange(
- &self,
- current: u64,
- new: u64,
- _success: Ordering,
- _failure: Ordering,
- ) -> Result<u64, u64> {
- let mut lock = self.inner.lock();
-
- if *lock == current {
- *lock = new;
- Ok(current)
- } else {
- Err(*lock)
- }
- }
-
- pub(crate) fn compare_exchange_weak(
- &self,
- current: u64,
- new: u64,
- success: Ordering,
- failure: Ordering,
- ) -> Result<u64, u64> {
- self.compare_exchange(current, new, success, failure)
- }
- }
+ #[path = "atomic_u64_as_mutex.rs"]
+ mod imp;
}
+
+pub(crate) use imp::{AtomicU64, StaticAtomicU64};
diff --git a/src/loom/std/atomic_u64_as_mutex.rs b/src/loom/std/atomic_u64_as_mutex.rs
new file mode 100644
index 0000000..9b3b6fa
--- /dev/null
+++ b/src/loom/std/atomic_u64_as_mutex.rs
@@ -0,0 +1,76 @@
+use crate::loom::sync::Mutex;
+use std::sync::atomic::Ordering;
+
+cfg_has_const_mutex_new! {
+ #[path = "atomic_u64_static_const_new.rs"]
+ mod static_macro;
+}
+
+cfg_not_has_const_mutex_new! {
+ #[path = "atomic_u64_static_once_cell.rs"]
+ mod static_macro;
+}
+
+pub(crate) use static_macro::StaticAtomicU64;
+
+#[derive(Debug)]
+pub(crate) struct AtomicU64 {
+ inner: Mutex<u64>,
+}
+
+impl AtomicU64 {
+ pub(crate) fn load(&self, _: Ordering) -> u64 {
+ *self.inner.lock()
+ }
+
+ pub(crate) fn store(&self, val: u64, _: Ordering) {
+ *self.inner.lock() = val;
+ }
+
+ pub(crate) fn fetch_add(&self, val: u64, _: Ordering) -> u64 {
+ let mut lock = self.inner.lock();
+ let prev = *lock;
+ *lock = prev + val;
+ prev
+ }
+
+ pub(crate) fn fetch_or(&self, val: u64, _: Ordering) -> u64 {
+ let mut lock = self.inner.lock();
+ let prev = *lock;
+ *lock = prev | val;
+ prev
+ }
+
+ pub(crate) fn compare_exchange(
+ &self,
+ current: u64,
+ new: u64,
+ _success: Ordering,
+ _failure: Ordering,
+ ) -> Result<u64, u64> {
+ let mut lock = self.inner.lock();
+
+ if *lock == current {
+ *lock = new;
+ Ok(current)
+ } else {
+ Err(*lock)
+ }
+ }
+
+ pub(crate) fn compare_exchange_weak(
+ &self,
+ current: u64,
+ new: u64,
+ success: Ordering,
+ failure: Ordering,
+ ) -> Result<u64, u64> {
+ self.compare_exchange(current, new, success, failure)
+ }
+}
+
+impl Default for AtomicU64 {
+ fn default() -> AtomicU64 {
+ AtomicU64::new(u64::default())
+ }
+}
diff --git a/src/loom/std/atomic_u64_native.rs b/src/loom/std/atomic_u64_native.rs
new file mode 100644
index 0000000..08adb28
--- /dev/null
+++ b/src/loom/std/atomic_u64_native.rs
@@ -0,0 +1,4 @@
+pub(crate) use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Alias `AtomicU64` to `StaticAtomicU64`
+pub(crate) type StaticAtomicU64 = AtomicU64;
diff --git a/src/loom/std/atomic_u64_static_const_new.rs b/src/loom/std/atomic_u64_static_const_new.rs
new file mode 100644
index 0000000..a421534
--- /dev/null
+++ b/src/loom/std/atomic_u64_static_const_new.rs
@@ -0,0 +1,12 @@
+use super::AtomicU64;
+use crate::loom::sync::Mutex;
+
+pub(crate) type StaticAtomicU64 = AtomicU64;
+
+impl AtomicU64 {
+ pub(crate) const fn new(val: u64) -> Self {
+ Self {
+ inner: Mutex::const_new(val),
+ }
+ }
+}
diff --git a/src/loom/std/atomic_u64_static_once_cell.rs b/src/loom/std/atomic_u64_static_once_cell.rs
new file mode 100644
index 0000000..40c6172
--- /dev/null
+++ b/src/loom/std/atomic_u64_static_once_cell.rs
@@ -0,0 +1,57 @@
+use super::AtomicU64;
+use crate::loom::sync::{atomic::Ordering, Mutex};
+use crate::util::once_cell::OnceCell;
+
+pub(crate) struct StaticAtomicU64 {
+ init: u64,
+ cell: OnceCell<Mutex<u64>>,
+}
+
+impl AtomicU64 {
+ pub(crate) fn new(val: u64) -> Self {
+ Self {
+ inner: Mutex::new(val),
+ }
+ }
+}
+
+impl StaticAtomicU64 {
+ pub(crate) const fn new(val: u64) -> StaticAtomicU64 {
+ StaticAtomicU64 {
+ init: val,
+ cell: OnceCell::new(),
+ }
+ }
+
+ pub(crate) fn load(&self, order: Ordering) -> u64 {
+ *self.inner().lock()
+ }
+
+ pub(crate) fn fetch_add(&self, val: u64, order: Ordering) -> u64 {
+ let mut lock = self.inner().lock();
+ let prev = *lock;
+ *lock = prev + val;
+ prev
+ }
+
+ pub(crate) fn compare_exchange_weak(
+ &self,
+ current: u64,
+ new: u64,
+ _success: Ordering,
+ _failure: Ordering,
+ ) -> Result<u64, u64> {
+ let mut lock = self.inner().lock();
+
+ if *lock == current {
+ *lock = new;
+ Ok(current)
+ } else {
+ Err(*lock)
+ }
+ }
+
+ fn inner(&self) -> &Mutex<u64> {
+ self.cell.get(|| Mutex::new(self.init))
+ }
+}
diff --git a/src/loom/std/atomic_u8.rs b/src/loom/std/atomic_u8.rs
deleted file mode 100644
index 408aea3..0000000
--- a/src/loom/std/atomic_u8.rs
+++ /dev/null
@@ -1,34 +0,0 @@
-use std::cell::UnsafeCell;
-use std::fmt;
-use std::ops::Deref;
-
-/// `AtomicU8` providing an additional `load_unsync` function.
-pub(crate) struct AtomicU8 {
- inner: UnsafeCell<std::sync::atomic::AtomicU8>,
-}
-
-unsafe impl Send for AtomicU8 {}
-unsafe impl Sync for AtomicU8 {}
-
-impl AtomicU8 {
- pub(crate) const fn new(val: u8) -> AtomicU8 {
- let inner = UnsafeCell::new(std::sync::atomic::AtomicU8::new(val));
- AtomicU8 { inner }
- }
-}
-
-impl Deref for AtomicU8 {
- type Target = std::sync::atomic::AtomicU8;
-
- fn deref(&self) -> &Self::Target {
- // safety: it is always safe to access `&self` fns on the inner value as
- // we never perform unsafe mutations.
- unsafe { &*self.inner.get() }
- }
-}
-
-impl fmt::Debug for AtomicU8 {
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- self.deref().fmt(fmt)
- }
-}
diff --git a/src/loom/std/atomic_usize.rs b/src/loom/std/atomic_usize.rs
index 0d5f36e..c5503a2 100644
--- a/src/loom/std/atomic_usize.rs
+++ b/src/loom/std/atomic_usize.rs
@@ -2,7 +2,7 @@
use std::fmt;
use std::ops;
-/// `AtomicUsize` providing an additional `load_unsync` function.
+/// `AtomicUsize` providing an additional `unsync_load` function.
pub(crate) struct AtomicUsize {
inner: UnsafeCell<std::sync::atomic::AtomicUsize>,
}
@@ -23,7 +23,7 @@
/// All mutations must have happened before the unsynchronized load.
/// Additionally, there must be no concurrent mutations.
pub(crate) unsafe fn unsync_load(&self) -> usize {
- *(*self.inner.get()).get_mut()
+ core::ptr::read(self.inner.get() as *const usize)
}
pub(crate) fn with_mut<R>(&mut self, f: impl FnOnce(&mut usize) -> R) -> R {
diff --git a/src/loom/std/mod.rs b/src/loom/std/mod.rs
index 8b6e8bc..6bd1ad9 100644
--- a/src/loom/std/mod.rs
+++ b/src/loom/std/mod.rs
@@ -1,10 +1,8 @@
#![cfg_attr(any(not(feature = "full"), loom), allow(unused_imports, dead_code))]
-mod atomic_ptr;
mod atomic_u16;
mod atomic_u32;
mod atomic_u64;
-mod atomic_u8;
mod atomic_usize;
mod mutex;
#[cfg(feature = "parking_lot")]
@@ -25,6 +23,10 @@
pub(crate) use crate::sync::AtomicWaker;
}
+pub(crate) mod hint {
+ pub(crate) use std::hint::spin_loop;
+}
+
pub(crate) mod rand {
use std::collections::hash_map::RandomState;
use std::hash::{BuildHasher, Hash, Hasher};
@@ -67,24 +69,39 @@
pub(crate) use crate::loom::std::mutex::Mutex;
pub(crate) mod atomic {
- pub(crate) use crate::loom::std::atomic_ptr::AtomicPtr;
pub(crate) use crate::loom::std::atomic_u16::AtomicU16;
pub(crate) use crate::loom::std::atomic_u32::AtomicU32;
- pub(crate) use crate::loom::std::atomic_u64::AtomicU64;
- pub(crate) use crate::loom::std::atomic_u8::AtomicU8;
+ pub(crate) use crate::loom::std::atomic_u64::{AtomicU64, StaticAtomicU64};
pub(crate) use crate::loom::std::atomic_usize::AtomicUsize;
- pub(crate) use std::sync::atomic::{fence, AtomicBool, Ordering};
- // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead.
- #[allow(deprecated)]
- pub(crate) use std::sync::atomic::spin_loop_hint;
+ pub(crate) use std::sync::atomic::{fence, AtomicBool, AtomicPtr, AtomicU8, Ordering};
}
}
pub(crate) mod sys {
#[cfg(feature = "rt-multi-thread")]
pub(crate) fn num_cpus() -> usize {
- usize::max(1, num_cpus::get())
+ const ENV_WORKER_THREADS: &str = "TOKIO_WORKER_THREADS";
+
+ match std::env::var(ENV_WORKER_THREADS) {
+ Ok(s) => {
+ let n = s.parse().unwrap_or_else(|e| {
+ panic!(
+ "\"{}\" must be usize, error: {}, value: {}",
+ ENV_WORKER_THREADS, e, s
+ )
+ });
+ assert!(n > 0, "\"{}\" cannot be set to 0", ENV_WORKER_THREADS);
+ n
+ }
+ Err(std::env::VarError::NotPresent) => usize::max(1, num_cpus::get()),
+ Err(std::env::VarError::NotUnicode(e)) => {
+ panic!(
+ "\"{}\" must be valid unicode, error: {:?}",
+ ENV_WORKER_THREADS, e
+ )
+ }
+ }
}
#[cfg(not(feature = "rt-multi-thread"))]
@@ -96,14 +113,12 @@
pub(crate) mod thread {
#[inline]
pub(crate) fn yield_now() {
- // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead.
- #[allow(deprecated)]
- std::sync::atomic::spin_loop_hint();
+ std::hint::spin_loop();
}
#[allow(unused_imports)]
pub(crate) use std::thread::{
- current, panicking, park, park_timeout, sleep, spawn, Builder, JoinHandle, LocalKey,
- Result, Thread, ThreadId,
+ current, panicking, park, park_timeout, sleep, spawn, AccessError, Builder, JoinHandle,
+ LocalKey, Result, Thread, ThreadId,
};
}
diff --git a/src/loom/std/mutex.rs b/src/loom/std/mutex.rs
index 3f686e0..076f786 100644
--- a/src/loom/std/mutex.rs
+++ b/src/loom/std/mutex.rs
@@ -13,6 +13,12 @@
}
#[inline]
+ #[cfg(not(tokio_no_const_mutex_new))]
+ pub(crate) const fn const_new(t: T) -> Mutex<T> {
+ Mutex(sync::Mutex::new(t))
+ }
+
+ #[inline]
pub(crate) fn lock(&self) -> MutexGuard<'_, T> {
match self.0.lock() {
Ok(guard) => guard,
diff --git a/src/loom/std/parking_lot.rs b/src/loom/std/parking_lot.rs
index 8448bed..e3af258 100644
--- a/src/loom/std/parking_lot.rs
+++ b/src/loom/std/parking_lot.rs
@@ -3,83 +3,143 @@
//!
//! This can be extended to additional types/methods as required.
+use std::fmt;
+use std::marker::PhantomData;
+use std::ops::{Deref, DerefMut};
use std::sync::LockResult;
use std::time::Duration;
+// All types in this file are marked with PhantomData to ensure that
+// parking_lot's send_guard feature does not leak through and affect when Tokio
+// types are Send.
+//
+// See <https://github.com/tokio-rs/tokio/pull/4359> for more info.
+
// Types that do not need wrapping
-pub(crate) use parking_lot::{MutexGuard, RwLockReadGuard, RwLockWriteGuard, WaitTimeoutResult};
-
-/// Adapter for `parking_lot::Mutex` to the `std::sync::Mutex` interface.
-#[derive(Debug)]
-pub(crate) struct Mutex<T: ?Sized>(parking_lot::Mutex<T>);
+pub(crate) use parking_lot::WaitTimeoutResult;
#[derive(Debug)]
-pub(crate) struct RwLock<T>(parking_lot::RwLock<T>);
+pub(crate) struct Mutex<T: ?Sized>(PhantomData<std::sync::Mutex<T>>, parking_lot::Mutex<T>);
-/// Adapter for `parking_lot::Condvar` to the `std::sync::Condvar` interface.
#[derive(Debug)]
-pub(crate) struct Condvar(parking_lot::Condvar);
+pub(crate) struct RwLock<T>(PhantomData<std::sync::RwLock<T>>, parking_lot::RwLock<T>);
+
+#[derive(Debug)]
+pub(crate) struct Condvar(PhantomData<std::sync::Condvar>, parking_lot::Condvar);
+
+#[derive(Debug)]
+pub(crate) struct MutexGuard<'a, T: ?Sized>(
+ PhantomData<std::sync::MutexGuard<'a, T>>,
+ parking_lot::MutexGuard<'a, T>,
+);
+
+#[derive(Debug)]
+pub(crate) struct RwLockReadGuard<'a, T: ?Sized>(
+ PhantomData<std::sync::RwLockReadGuard<'a, T>>,
+ parking_lot::RwLockReadGuard<'a, T>,
+);
+
+#[derive(Debug)]
+pub(crate) struct RwLockWriteGuard<'a, T: ?Sized>(
+ PhantomData<std::sync::RwLockWriteGuard<'a, T>>,
+ parking_lot::RwLockWriteGuard<'a, T>,
+);
impl<T> Mutex<T> {
#[inline]
pub(crate) fn new(t: T) -> Mutex<T> {
- Mutex(parking_lot::Mutex::new(t))
+ Mutex(PhantomData, parking_lot::Mutex::new(t))
}
#[inline]
- #[cfg(all(feature = "parking_lot", not(all(loom, test)),))]
+ #[cfg(all(feature = "parking_lot", not(all(loom, test))))]
#[cfg_attr(docsrs, doc(cfg(all(feature = "parking_lot",))))]
pub(crate) const fn const_new(t: T) -> Mutex<T> {
- Mutex(parking_lot::const_mutex(t))
+ Mutex(PhantomData, parking_lot::const_mutex(t))
}
#[inline]
pub(crate) fn lock(&self) -> MutexGuard<'_, T> {
- self.0.lock()
+ MutexGuard(PhantomData, self.1.lock())
}
#[inline]
pub(crate) fn try_lock(&self) -> Option<MutexGuard<'_, T>> {
- self.0.try_lock()
+ self.1
+ .try_lock()
+ .map(|guard| MutexGuard(PhantomData, guard))
}
#[inline]
pub(crate) fn get_mut(&mut self) -> &mut T {
- self.0.get_mut()
+ self.1.get_mut()
}
// Note: Additional methods `is_poisoned` and `into_inner`, can be
// provided here as needed.
}
+impl<'a, T: ?Sized> Deref for MutexGuard<'a, T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.1.deref()
+ }
+}
+
+impl<'a, T: ?Sized> DerefMut for MutexGuard<'a, T> {
+ fn deref_mut(&mut self) -> &mut T {
+ self.1.deref_mut()
+ }
+}
+
impl<T> RwLock<T> {
pub(crate) fn new(t: T) -> RwLock<T> {
- RwLock(parking_lot::RwLock::new(t))
+ RwLock(PhantomData, parking_lot::RwLock::new(t))
}
pub(crate) fn read(&self) -> LockResult<RwLockReadGuard<'_, T>> {
- Ok(self.0.read())
+ Ok(RwLockReadGuard(PhantomData, self.1.read()))
}
pub(crate) fn write(&self) -> LockResult<RwLockWriteGuard<'_, T>> {
- Ok(self.0.write())
+ Ok(RwLockWriteGuard(PhantomData, self.1.write()))
+ }
+}
+
+impl<'a, T: ?Sized> Deref for RwLockReadGuard<'a, T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.1.deref()
+ }
+}
+
+impl<'a, T: ?Sized> Deref for RwLockWriteGuard<'a, T> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ self.1.deref()
+ }
+}
+
+impl<'a, T: ?Sized> DerefMut for RwLockWriteGuard<'a, T> {
+ fn deref_mut(&mut self) -> &mut T {
+ self.1.deref_mut()
}
}
impl Condvar {
#[inline]
pub(crate) fn new() -> Condvar {
- Condvar(parking_lot::Condvar::new())
+ Condvar(PhantomData, parking_lot::Condvar::new())
}
#[inline]
pub(crate) fn notify_one(&self) {
- self.0.notify_one();
+ self.1.notify_one();
}
#[inline]
pub(crate) fn notify_all(&self) {
- self.0.notify_all();
+ self.1.notify_all();
}
#[inline]
@@ -87,7 +147,7 @@
&self,
mut guard: MutexGuard<'a, T>,
) -> LockResult<MutexGuard<'a, T>> {
- self.0.wait(&mut guard);
+ self.1.wait(&mut guard.1);
Ok(guard)
}
@@ -97,10 +157,28 @@
mut guard: MutexGuard<'a, T>,
timeout: Duration,
) -> LockResult<(MutexGuard<'a, T>, WaitTimeoutResult)> {
- let wtr = self.0.wait_for(&mut guard, timeout);
+ let wtr = self.1.wait_for(&mut guard.1, timeout);
Ok((guard, wtr))
}
// Note: Additional methods `wait_timeout_ms`, `wait_timeout_until`,
// `wait_until` can be provided here as needed.
}
+
+impl<'a, T: ?Sized + fmt::Display> fmt::Display for MutexGuard<'a, T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Display::fmt(&self.1, f)
+ }
+}
+
+impl<'a, T: ?Sized + fmt::Display> fmt::Display for RwLockReadGuard<'a, T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Display::fmt(&self.1, f)
+ }
+}
+
+impl<'a, T: ?Sized + fmt::Display> fmt::Display for RwLockWriteGuard<'a, T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Display::fmt(&self.1, f)
+ }
+}
diff --git a/src/macros/addr_of.rs b/src/macros/addr_of.rs
new file mode 100644
index 0000000..51d4889
--- /dev/null
+++ b/src/macros/addr_of.rs
@@ -0,0 +1,53 @@
+//! This module defines a macro that lets you go from a raw pointer to a struct
+//! to a raw pointer to a field of the struct.
+
+#[cfg(not(tokio_no_addr_of))]
+macro_rules! generate_addr_of_methods {
+ (
+ impl<$($gen:ident)*> $struct_name:ty {$(
+ $(#[$attrs:meta])*
+ $vis:vis unsafe fn $fn_name:ident(self: NonNull<Self>) -> NonNull<$field_type:ty> {
+ &self$(.$field_name:tt)+
+ }
+ )*}
+ ) => {
+ impl<$($gen)*> $struct_name {$(
+ $(#[$attrs])*
+ $vis unsafe fn $fn_name(me: ::core::ptr::NonNull<Self>) -> ::core::ptr::NonNull<$field_type> {
+ let me = me.as_ptr();
+ let field = ::std::ptr::addr_of_mut!((*me) $(.$field_name)+ );
+ ::core::ptr::NonNull::new_unchecked(field)
+ }
+ )*}
+ };
+}
+
+// The `addr_of_mut!` macro is only available for MSRV at least 1.51.0. This
+// version of the macro uses a workaround for older versions of rustc.
+#[cfg(tokio_no_addr_of)]
+macro_rules! generate_addr_of_methods {
+ (
+ impl<$($gen:ident)*> $struct_name:ty {$(
+ $(#[$attrs:meta])*
+ $vis:vis unsafe fn $fn_name:ident(self: NonNull<Self>) -> NonNull<$field_type:ty> {
+ &self$(.$field_name:tt)+
+ }
+ )*}
+ ) => {
+ impl<$($gen)*> $struct_name {$(
+ $(#[$attrs])*
+ $vis unsafe fn $fn_name(me: ::core::ptr::NonNull<Self>) -> ::core::ptr::NonNull<$field_type> {
+ let me = me.as_ptr();
+ let me_u8 = me as *mut u8;
+
+ let field_offset = {
+ let me_ref = &*me;
+ let field_ref_u8 = (&me_ref $(.$field_name)+ ) as *const $field_type as *const u8;
+ field_ref_u8.offset_from(me_u8)
+ };
+
+ ::core::ptr::NonNull::new_unchecked(me_u8.offset(field_offset).cast())
+ }
+ )*}
+ };
+}
diff --git a/src/macros/cfg.rs b/src/macros/cfg.rs
index 606bce7..1c66d24 100644
--- a/src/macros/cfg.rs
+++ b/src/macros/cfg.rs
@@ -61,6 +61,7 @@
($($item:item)*) => {
$(
#[cfg(feature = "fs")]
+ #[cfg(not(tokio_wasi))]
#[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
$item
)*
@@ -69,7 +70,11 @@
macro_rules! cfg_io_blocking {
($($item:item)*) => {
- $( #[cfg(any(feature = "io-std", feature = "fs"))] $item )*
+ $( #[cfg(any(
+ feature = "io-std",
+ feature = "fs",
+ all(windows, feature = "process"),
+ ))] $item )*
}
}
@@ -78,12 +83,12 @@
$(
#[cfg(any(
feature = "net",
- feature = "process",
+ all(unix, feature = "process"),
all(unix, feature = "signal"),
))]
#[cfg_attr(docsrs, doc(cfg(any(
feature = "net",
- feature = "process",
+ all(unix, feature = "process"),
all(unix, feature = "signal"),
))))]
$item
@@ -96,10 +101,9 @@
$(
#[cfg(any(
feature = "net",
- feature = "process",
+ all(unix, feature = "process"),
all(unix, feature = "signal"),
))]
- #[cfg_attr(docsrs, doc(cfg(all())))]
$item
)*
}
@@ -110,7 +114,7 @@
$(
#[cfg(not(any(
feature = "net",
- feature = "process",
+ all(unix, feature = "process"),
all(unix, feature = "signal"),
)))]
$item
@@ -175,20 +179,38 @@
}
}
-macro_rules! cfg_stats {
+macro_rules! cfg_metrics {
($($item:item)*) => {
$(
- #[cfg(all(tokio_unstable, feature = "stats"))]
- #[cfg_attr(docsrs, doc(cfg(feature = "stats")))]
+ // For now, metrics is only disabled in loom tests.
+ // When stabilized, it might have a dedicated feature flag.
+ #[cfg(all(tokio_unstable, not(loom)))]
+ #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
$item
)*
}
}
-macro_rules! cfg_not_stats {
+macro_rules! cfg_not_metrics {
($($item:item)*) => {
$(
- #[cfg(not(all(tokio_unstable, feature = "stats")))]
+ #[cfg(not(all(tokio_unstable, not(loom))))]
+ $item
+ )*
+ }
+}
+
+macro_rules! cfg_not_rt_and_metrics_and_net {
+ ($($item:item)*) => {
+ $( #[cfg(not(all(feature = "net", feature = "rt", all(tokio_unstable, not(loom)))))]$item )*
+ }
+}
+
+macro_rules! cfg_net_or_process {
+ ($($item:item)*) => {
+ $(
+ #[cfg(any(feature = "net", feature = "process"))]
+ #[cfg_attr(docsrs, doc(cfg(any(feature = "net", feature = "process"))))]
$item
)*
}
@@ -230,6 +252,7 @@
#[cfg(feature = "process")]
#[cfg_attr(docsrs, doc(cfg(feature = "process")))]
#[cfg(not(loom))]
+ #[cfg(not(tokio_wasi))]
$item
)*
}
@@ -258,6 +281,7 @@
#[cfg(feature = "signal")]
#[cfg_attr(docsrs, doc(cfg(feature = "signal")))]
#[cfg(not(loom))]
+ #[cfg(not(tokio_wasi))]
$item
)*
}
@@ -273,6 +297,13 @@
}
}
+macro_rules! cfg_signal_internal_and_unix {
+ ($($item:item)*) => {
+ #[cfg(unix)]
+ cfg_signal_internal! { $($item)* }
+ }
+}
+
macro_rules! cfg_not_signal_internal {
($($item:item)*) => {
$(
@@ -317,7 +348,7 @@
macro_rules! cfg_rt_multi_thread {
($($item:item)*) => {
$(
- #[cfg(feature = "rt-multi-thread")]
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
#[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
$item
)*
@@ -366,10 +397,20 @@
($($item:item)*) => {
$(
#[cfg(all(tokio_unstable, feature = "tracing"))]
- #[cfg_attr(docsrs, doc(cfg(feature = "tracing")))]
+ #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))]
$item
)*
- }
+ };
+}
+
+macro_rules! cfg_unstable {
+ ($($item:item)*) => {
+ $(
+ #[cfg(tokio_unstable)]
+ #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
+ $item
+ )*
+ };
}
macro_rules! cfg_not_trace {
@@ -420,12 +461,14 @@
macro_rules! cfg_has_atomic_u64 {
($($item:item)*) => {
$(
- #[cfg(not(any(
- target_arch = "arm",
- target_arch = "mips",
- target_arch = "powerpc",
- target_arch = "riscv32"
- )))]
+ #[cfg_attr(
+ not(tokio_no_target_has_atomic),
+ cfg(all(target_has_atomic = "64", not(tokio_no_atomic_u64))
+ ))]
+ #[cfg_attr(
+ tokio_no_target_has_atomic,
+ cfg(not(tokio_no_atomic_u64))
+ )]
$item
)*
}
@@ -434,12 +477,62 @@
macro_rules! cfg_not_has_atomic_u64 {
($($item:item)*) => {
$(
- #[cfg(any(
- target_arch = "arm",
- target_arch = "mips",
- target_arch = "powerpc",
- target_arch = "riscv32"
- ))]
+ #[cfg_attr(
+ not(tokio_no_target_has_atomic),
+ cfg(any(not(target_has_atomic = "64"), tokio_no_atomic_u64)
+ ))]
+ #[cfg_attr(
+ tokio_no_target_has_atomic,
+ cfg(tokio_no_atomic_u64)
+ )]
+ $item
+ )*
+ }
+}
+
+macro_rules! cfg_has_const_mutex_new {
+ ($($item:item)*) => {
+ $(
+ #[cfg(all(
+ not(all(loom, test)),
+ any(
+ feature = "parking_lot",
+ not(tokio_no_const_mutex_new)
+ )
+ ))]
+ $item
+ )*
+ }
+}
+
+macro_rules! cfg_not_has_const_mutex_new {
+ ($($item:item)*) => {
+ $(
+ #[cfg(not(all(
+ not(all(loom, test)),
+ any(
+ feature = "parking_lot",
+ not(tokio_no_const_mutex_new)
+ )
+ )))]
+ $item
+ )*
+ }
+}
+
+macro_rules! cfg_not_wasi {
+ ($($item:item)*) => {
+ $(
+ #[cfg(not(tokio_wasi))]
+ $item
+ )*
+ }
+}
+
+macro_rules! cfg_is_wasm_not_wasi {
+ ($($item:item)*) => {
+ $(
+ #[cfg(tokio_wasm_not_wasi)]
$item
)*
}
diff --git a/src/macros/join.rs b/src/macros/join.rs
index f91b5f1..7e85203 100644
--- a/src/macros/join.rs
+++ b/src/macros/join.rs
@@ -12,7 +12,7 @@
/// for **all** branches complete regardless if any complete with `Err`. Use
/// [`try_join!`] to return early when `Err` is encountered.
///
-/// [`try_join!`]: macro@try_join
+/// [`try_join!`]: crate::try_join
///
/// # Notes
///
@@ -60,6 +60,9 @@
// normalization is complete.
( $($count:tt)* )
+ // The expression `0+1+1+ ... +1` equal to the number of branches.
+ ( $($total:tt)* )
+
// Normalized join! branches
$( ( $($skip:tt)* ) $e:expr, )*
@@ -69,24 +72,66 @@
// Safety: nothing must be moved out of `futures`. This is to satisfy
// the requirement of `Pin::new_unchecked` called below.
+ //
+ // We can't use the `pin!` macro for this because `futures` is a tuple
+ // and the standard library provides no way to pin-project to the fields
+ // of a tuple.
let mut futures = ( $( maybe_done($e), )* );
+ // This assignment makes sure that the `poll_fn` closure only has a
+ // reference to the futures, instead of taking ownership of them. This
+ // mitigates the issue described in
+ // <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484>
+ let mut futures = &mut futures;
+
+ // Each time the future created by poll_fn is polled, a different future will be polled first
+ // to ensure every future passed to join! gets a chance to make progress even if
+ // one of the futures consumes the whole budget.
+ //
+ // This is number of futures that will be skipped in the first loop
+ // iteration the next time.
+ let mut skip_next_time: u32 = 0;
+
poll_fn(move |cx| {
+ const COUNT: u32 = $($total)*;
+
let mut is_pending = false;
+ let mut to_run = COUNT;
+
+ // The number of futures that will be skipped in the first loop iteration.
+ let mut skip = skip_next_time;
+
+ skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 };
+
+ // This loop runs twice and the first `skip` futures
+ // are not polled in the first iteration.
+ loop {
$(
- // Extract the future for this branch from the tuple.
- let ( $($skip,)* fut, .. ) = &mut futures;
+ if skip == 0 {
+ if to_run == 0 {
+ // Every future has been polled
+ break;
+ }
+ to_run -= 1;
- // Safety: future is stored on the stack above
- // and never moved.
- let mut fut = unsafe { Pin::new_unchecked(fut) };
+ // Extract the future for this branch from the tuple.
+ let ( $($skip,)* fut, .. ) = &mut *futures;
- // Try polling
- if fut.poll(cx).is_pending() {
- is_pending = true;
+ // Safety: future is stored on the stack above
+ // and never moved.
+ let mut fut = unsafe { Pin::new_unchecked(fut) };
+
+ // Try polling
+ if fut.poll(cx).is_pending() {
+ is_pending = true;
+ }
+ } else {
+ // Future skipped, one less future to skip in the next iteration
+ skip -= 1;
}
)*
+ }
if is_pending {
Pending
@@ -107,13 +152,13 @@
// ===== Normalize =====
- (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
- $crate::join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*)
+ (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
+ $crate::join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*)
};
// ===== Entry point =====
( $($e:expr),* $(,)?) => {
- $crate::join!(@{ () } $($e,)*)
+ $crate::join!(@{ () (0) } $($e,)*)
};
}
diff --git a/src/macros/mod.rs b/src/macros/mod.rs
index a1839c8..57678c6 100644
--- a/src/macros/mod.rs
+++ b/src/macros/mod.rs
@@ -15,6 +15,9 @@
#[macro_use]
mod thread_local;
+#[macro_use]
+mod addr_of;
+
cfg_trace! {
#[macro_use]
mod trace;
diff --git a/src/macros/scoped_tls.rs b/src/macros/scoped_tls.rs
index f2504cb..ed74b32 100644
--- a/src/macros/scoped_tls.rs
+++ b/src/macros/scoped_tls.rs
@@ -10,7 +10,7 @@
$vis static $name: $crate::macros::scoped_tls::ScopedKey<$ty>
= $crate::macros::scoped_tls::ScopedKey {
inner: {
- thread_local!(static FOO: ::std::cell::Cell<*const ()> = {
+ tokio_thread_local!(static FOO: ::std::cell::Cell<*const ()> = const {
std::cell::Cell::new(::std::ptr::null())
});
&FOO
diff --git a/src/macros/select.rs b/src/macros/select.rs
index 051f8cb..7ba04a0 100644
--- a/src/macros/select.rs
+++ b/src/macros/select.rs
@@ -101,6 +101,7 @@
/// * [`tokio::sync::watch::Receiver::changed`](crate::sync::watch::Receiver::changed)
/// * [`tokio::net::TcpListener::accept`](crate::net::TcpListener::accept)
/// * [`tokio::net::UnixListener::accept`](crate::net::UnixListener::accept)
+/// * [`tokio::signal::unix::Signal::recv`](crate::signal::unix::Signal::recv)
/// * [`tokio::io::AsyncReadExt::read`](crate::io::AsyncReadExt::read) on any `AsyncRead`
/// * [`tokio::io::AsyncReadExt::read_buf`](crate::io::AsyncReadExt::read_buf) on any `AsyncRead`
/// * [`tokio::io::AsyncWriteExt::write`](crate::io::AsyncWriteExt::write) on any `AsyncWrite`
@@ -429,7 +430,8 @@
//
// This module is defined within a scope and should not leak out of this
// macro.
- mod util {
+ #[doc(hidden)]
+ mod __tokio_select_util {
// Generate an enum with one variant per select branch
$crate::select_priv_declare_output_enum!( ( $($count)* ) );
}
@@ -442,13 +444,13 @@
const BRANCHES: u32 = $crate::count!( $($count)* );
- let mut disabled: util::Mask = Default::default();
+ let mut disabled: __tokio_select_util::Mask = Default::default();
// First, invoke all the pre-conditions. For any that return true,
// set the appropriate bit in `disabled`.
$(
if !$c {
- let mask: util::Mask = 1 << $crate::count!( $($skip)* );
+ let mask: __tokio_select_util::Mask = 1 << $crate::count!( $($skip)* );
disabled |= mask;
}
)*
@@ -458,8 +460,18 @@
let mut output = {
// Safety: Nothing must be moved out of `futures`. This is to
// satisfy the requirement of `Pin::new_unchecked` called below.
+ //
+ // We can't use the `pin!` macro for this because `futures` is a
+ // tuple and the standard library provides no way to pin-project to
+ // the fields of a tuple.
let mut futures = ( $( $fut , )+ );
+ // This assignment makes sure that the `poll_fn` closure only has a
+ // reference to the futures, instead of taking ownership of them.
+ // This mitigates the issue described in
+ // <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484>
+ let mut futures = &mut futures;
+
$crate::macros::support::poll_fn(|cx| {
// Track if any branch returns pending. If no branch completes
// **or** returns pending, this implies that all branches are
@@ -495,7 +507,7 @@
// Extract the future for this branch from the
// tuple
- let ( $($skip,)* fut, .. ) = &mut futures;
+ let ( $($skip,)* fut, .. ) = &mut *futures;
// Safety: future is stored on the stack above
// and never moved.
@@ -525,7 +537,7 @@
}
// The select is complete, return the value
- return Ready($crate::select_variant!(util::Out, ($($skip)*))(out));
+ return Ready($crate::select_variant!(__tokio_select_util::Out, ($($skip)*))(out));
}
)*
_ => unreachable!("reaching this means there probably is an off by one bug"),
@@ -536,16 +548,16 @@
Pending
} else {
// All branches have been disabled.
- Ready(util::Out::Disabled)
+ Ready(__tokio_select_util::Out::Disabled)
}
}).await
};
match output {
$(
- $crate::select_variant!(util::Out, ($($skip)*) ($bind)) => $handle,
+ $crate::select_variant!(__tokio_select_util::Out, ($($skip)*) ($bind)) => $handle,
)*
- util::Out::Disabled => $else,
+ __tokio_select_util::Out::Disabled => $else,
_ => unreachable!("failed to match bind"),
}
}};
@@ -801,6 +813,9 @@
(_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _) => {
63
};
+ (_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _) => {
+ 64
+ };
}
#[macro_export]
diff --git a/src/macros/support.rs b/src/macros/support.rs
index 7f11bc6..10526bc 100644
--- a/src/macros/support.rs
+++ b/src/macros/support.rs
@@ -1,7 +1,11 @@
cfg_macros! {
pub use crate::future::poll_fn;
pub use crate::future::maybe_done::maybe_done;
- pub use crate::util::thread_rng_n;
+
+ #[doc(hidden)]
+ pub fn thread_rng_n(n: u32) -> u32 {
+ crate::runtime::context::thread_rng_n(n)
+ }
}
pub use std::future::Future;
diff --git a/src/macros/thread_local.rs b/src/macros/thread_local.rs
index d848947..74be99a 100644
--- a/src/macros/thread_local.rs
+++ b/src/macros/thread_local.rs
@@ -1,4 +1,32 @@
#[cfg(all(loom, test))]
-macro_rules! thread_local {
+macro_rules! tokio_thread_local {
+ ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => {
+ loom::thread_local! {
+ $(#[$attrs])*
+ $vis static $name: $ty = $expr;
+ }
+ };
+
($($tts:tt)+) => { loom::thread_local!{ $($tts)+ } }
}
+
+#[cfg(not(tokio_no_const_thread_local))]
+#[cfg(not(all(loom, test)))]
+macro_rules! tokio_thread_local {
+ ($($tts:tt)+) => {
+ ::std::thread_local!{ $($tts)+ }
+ }
+}
+
+#[cfg(tokio_no_const_thread_local)]
+#[cfg(not(all(loom, test)))]
+macro_rules! tokio_thread_local {
+ ($(#[$attrs:meta])* $vis:vis static $name:ident: $ty:ty = const { $expr:expr } $(;)?) => {
+ ::std::thread_local! {
+ $(#[$attrs])*
+ $vis static $name: $ty = $expr;
+ }
+ };
+
+ ($($tts:tt)+) => { ::std::thread_local!{ $($tts)+ } }
+}
diff --git a/src/macros/trace.rs b/src/macros/trace.rs
index 31dde2f..80a257e 100644
--- a/src/macros/trace.rs
+++ b/src/macros/trace.rs
@@ -1,9 +1,8 @@
cfg_trace! {
macro_rules! trace_op {
- ($name:literal, $readiness:literal, $parent:expr) => {
+ ($name:expr, $readiness:literal) => {
tracing::trace!(
target: "runtime::resource::poll_op",
- parent: $parent,
op_name = $name,
is_ready = $readiness
);
@@ -11,14 +10,14 @@
}
macro_rules! trace_poll_op {
- ($name:literal, $poll:expr, $parent:expr $(,)*) => {
+ ($name:expr, $poll:expr $(,)*) => {
match $poll {
std::task::Poll::Ready(t) => {
- trace_op!($name, true, $parent);
+ trace_op!($name, true);
std::task::Poll::Ready(t)
}
std::task::Poll::Pending => {
- trace_op!($name, false, $parent);
+ trace_op!($name, false);
return std::task::Poll::Pending;
}
}
diff --git a/src/macros/try_join.rs b/src/macros/try_join.rs
index 6d3a893..597cd5d 100644
--- a/src/macros/try_join.rs
+++ b/src/macros/try_join.rs
@@ -106,6 +106,9 @@
// normalization is complete.
( $($count:tt)* )
+ // The expression `0+1+1+ ... +1` equal to the number of branches.
+ ( $($total:tt)* )
+
// Normalized try_join! branches
$( ( $($skip:tt)* ) $e:expr, )*
@@ -115,26 +118,68 @@
// Safety: nothing must be moved out of `futures`. This is to satisfy
// the requirement of `Pin::new_unchecked` called below.
+ //
+ // We can't use the `pin!` macro for this because `futures` is a tuple
+ // and the standard library provides no way to pin-project to the fields
+ // of a tuple.
let mut futures = ( $( maybe_done($e), )* );
+ // This assignment makes sure that the `poll_fn` closure only has a
+ // reference to the futures, instead of taking ownership of them. This
+ // mitigates the issue described in
+ // <https://internals.rust-lang.org/t/surprising-soundness-trouble-around-pollfn/17484>
+ let mut futures = &mut futures;
+
+ // Each time the future created by poll_fn is polled, a different future will be polled first
+ // to ensure every future passed to join! gets a chance to make progress even if
+ // one of the futures consumes the whole budget.
+ //
+ // This is number of futures that will be skipped in the first loop
+ // iteration the next time.
+ let mut skip_next_time: u32 = 0;
+
poll_fn(move |cx| {
+ const COUNT: u32 = $($total)*;
+
let mut is_pending = false;
+ let mut to_run = COUNT;
+
+ // The number of futures that will be skipped in the first loop iteration
+ let mut skip = skip_next_time;
+
+ skip_next_time = if skip + 1 == COUNT { 0 } else { skip + 1 };
+
+ // This loop runs twice and the first `skip` futures
+ // are not polled in the first iteration.
+ loop {
$(
- // Extract the future for this branch from the tuple.
- let ( $($skip,)* fut, .. ) = &mut futures;
+ if skip == 0 {
+ if to_run == 0 {
+ // Every future has been polled
+ break;
+ }
+ to_run -= 1;
- // Safety: future is stored on the stack above
- // and never moved.
- let mut fut = unsafe { Pin::new_unchecked(fut) };
+ // Extract the future for this branch from the tuple.
+ let ( $($skip,)* fut, .. ) = &mut *futures;
- // Try polling
- if fut.as_mut().poll(cx).is_pending() {
- is_pending = true;
- } else if fut.as_mut().output_mut().expect("expected completed future").is_err() {
- return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap()))
+ // Safety: future is stored on the stack above
+ // and never moved.
+ let mut fut = unsafe { Pin::new_unchecked(fut) };
+
+ // Try polling
+ if fut.as_mut().poll(cx).is_pending() {
+ is_pending = true;
+ } else if fut.as_mut().output_mut().expect("expected completed future").is_err() {
+ return Ready(Err(fut.take_output().expect("expected completed future").err().unwrap()))
+ }
+ } else {
+ // Future skipped, one less future to skip in the next iteration
+ skip -= 1;
}
)*
+ }
if is_pending {
Pending
@@ -159,13 +204,13 @@
// ===== Normalize =====
- (@ { ( $($s:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
- $crate::try_join!(@{ ($($s)* _) $($t)* ($($s)*) $e, } $($r)*)
+ (@ { ( $($s:tt)* ) ( $($n:tt)* ) $($t:tt)* } $e:expr, $($r:tt)* ) => {
+ $crate::try_join!(@{ ($($s)* _) ($($n)* + 1) $($t)* ($($s)*) $e, } $($r)*)
};
// ===== Entry point =====
( $($e:expr),* $(,)?) => {
- $crate::try_join!(@{ () } $($e,)*)
+ $crate::try_join!(@{ () (0) } $($e,)*)
};
}
diff --git a/src/net/addr.rs b/src/net/addr.rs
index ec4fa19..fb8248f 100644
--- a/src/net/addr.rs
+++ b/src/net/addr.rs
@@ -1,5 +1,4 @@
-use crate::future;
-
+use std::future;
use std::io;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
@@ -56,7 +55,7 @@
fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future {
let iter = Some(*self).into_iter();
- future::ok(iter)
+ future::ready(Ok(iter))
}
}
@@ -96,7 +95,7 @@
fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future {
let iter = Some(SocketAddr::from(*self)).into_iter();
- future::ok(iter)
+ future::ready(Ok(iter))
}
}
@@ -137,8 +136,23 @@
type Future = ReadyFuture<Self::Iter>;
fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future {
- let iter = self.to_vec().into_iter();
- future::ok(iter)
+ #[inline]
+ fn slice_to_vec(addrs: &[SocketAddr]) -> Vec<SocketAddr> {
+ addrs.to_vec()
+ }
+
+ // This uses a helper method because clippy doesn't like the `to_vec()`
+ // call here (it will allocate, whereas `self.iter().copied()` would
+ // not), but it's actually necessary in order to ensure that the
+ // returned iterator is valid for the `'static` lifetime, which the
+ // borrowed `slice::Iter` iterator would not be.
+ //
+ // Note that we can't actually add an `allow` attribute for
+ // `clippy::unnecessary_to_owned` here, as Tokio's CI runs clippy lints
+ // on Rust 1.52 to avoid breaking LTS releases of Tokio. Users of newer
+ // Rust versions who see this lint should just ignore it.
+ let iter = slice_to_vec(self).into_iter();
+ future::ready(Ok(iter))
}
}
@@ -230,7 +244,7 @@
type Future = <str as sealed::ToSocketAddrsPriv>::Future;
fn to_socket_addrs(&self, _: sealed::Internal) -> Self::Future {
- (&self[..]).to_socket_addrs(sealed::Internal)
+ self[..].to_socket_addrs(sealed::Internal)
}
}
}
diff --git a/src/net/mod.rs b/src/net/mod.rs
index 0b8c1ec..2d317a8 100644
--- a/src/net/mod.rs
+++ b/src/net/mod.rs
@@ -23,8 +23,10 @@
//! [`UnixDatagram`]: UnixDatagram
mod addr;
-#[cfg(feature = "net")]
-pub(crate) use addr::to_socket_addrs;
+cfg_not_wasi! {
+ #[cfg(feature = "net")]
+ pub(crate) use addr::to_socket_addrs;
+}
pub use addr::ToSocketAddrs;
cfg_net! {
@@ -33,11 +35,13 @@
pub mod tcp;
pub use tcp::listener::TcpListener;
- pub use tcp::socket::TcpSocket;
pub use tcp::stream::TcpStream;
+ cfg_not_wasi! {
+ pub use tcp::socket::TcpSocket;
- mod udp;
- pub use udp::UdpSocket;
+ mod udp;
+ pub use udp::UdpSocket;
+ }
}
cfg_net_unix! {
diff --git a/src/net/tcp/listener.rs b/src/net/tcp/listener.rs
index 8aecb21..4441313 100644
--- a/src/net/tcp/listener.rs
+++ b/src/net/tcp/listener.rs
@@ -1,6 +1,9 @@
use crate::io::{Interest, PollEvented};
use crate::net::tcp::TcpStream;
-use crate::net::{to_socket_addrs, ToSocketAddrs};
+
+cfg_not_wasi! {
+ use crate::net::{to_socket_addrs, ToSocketAddrs};
+}
use std::convert::TryFrom;
use std::fmt;
@@ -55,68 +58,70 @@
}
impl TcpListener {
- /// Creates a new TcpListener, which will be bound to the specified address.
- ///
- /// The returned listener is ready for accepting connections.
- ///
- /// Binding with a port number of 0 will request that the OS assigns a port
- /// to this listener. The port allocated can be queried via the `local_addr`
- /// method.
- ///
- /// The address type can be any implementor of the [`ToSocketAddrs`] trait.
- /// If `addr` yields multiple addresses, bind will be attempted with each of
- /// the addresses until one succeeds and returns the listener. If none of
- /// the addresses succeed in creating a listener, the error returned from
- /// the last attempt (the last address) is returned.
- ///
- /// This function sets the `SO_REUSEADDR` option on the socket.
- ///
- /// To configure the socket before binding, you can use the [`TcpSocket`]
- /// type.
- ///
- /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs
- /// [`TcpSocket`]: struct@crate::net::TcpSocket
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use tokio::net::TcpListener;
- ///
- /// use std::io;
- ///
- /// #[tokio::main]
- /// async fn main() -> io::Result<()> {
- /// let listener = TcpListener::bind("127.0.0.1:2345").await?;
- ///
- /// // use the listener
- ///
- /// # let _ = listener;
- /// Ok(())
- /// }
- /// ```
- pub async fn bind<A: ToSocketAddrs>(addr: A) -> io::Result<TcpListener> {
- let addrs = to_socket_addrs(addr).await?;
+ cfg_not_wasi! {
+ /// Creates a new TcpListener, which will be bound to the specified address.
+ ///
+ /// The returned listener is ready for accepting connections.
+ ///
+ /// Binding with a port number of 0 will request that the OS assigns a port
+ /// to this listener. The port allocated can be queried via the `local_addr`
+ /// method.
+ ///
+ /// The address type can be any implementor of the [`ToSocketAddrs`] trait.
+ /// If `addr` yields multiple addresses, bind will be attempted with each of
+ /// the addresses until one succeeds and returns the listener. If none of
+ /// the addresses succeed in creating a listener, the error returned from
+ /// the last attempt (the last address) is returned.
+ ///
+ /// This function sets the `SO_REUSEADDR` option on the socket.
+ ///
+ /// To configure the socket before binding, you can use the [`TcpSocket`]
+ /// type.
+ ///
+ /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs
+ /// [`TcpSocket`]: struct@crate::net::TcpSocket
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::TcpListener;
+ ///
+ /// use std::io;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> io::Result<()> {
+ /// let listener = TcpListener::bind("127.0.0.1:2345").await?;
+ ///
+ /// // use the listener
+ ///
+ /// # let _ = listener;
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn bind<A: ToSocketAddrs>(addr: A) -> io::Result<TcpListener> {
+ let addrs = to_socket_addrs(addr).await?;
- let mut last_err = None;
+ let mut last_err = None;
- for addr in addrs {
- match TcpListener::bind_addr(addr) {
- Ok(listener) => return Ok(listener),
- Err(e) => last_err = Some(e),
+ for addr in addrs {
+ match TcpListener::bind_addr(addr) {
+ Ok(listener) => return Ok(listener),
+ Err(e) => last_err = Some(e),
+ }
}
+
+ Err(last_err.unwrap_or_else(|| {
+ io::Error::new(
+ io::ErrorKind::InvalidInput,
+ "could not resolve to any address",
+ )
+ }))
}
- Err(last_err.unwrap_or_else(|| {
- io::Error::new(
- io::ErrorKind::InvalidInput,
- "could not resolve to any address",
- )
- }))
- }
-
- fn bind_addr(addr: SocketAddr) -> io::Result<TcpListener> {
- let listener = mio::net::TcpListener::bind(addr)?;
- TcpListener::new(listener)
+ fn bind_addr(addr: SocketAddr) -> io::Result<TcpListener> {
+ let listener = mio::net::TcpListener::bind(addr)?;
+ TcpListener::new(listener)
+ }
}
/// Accepts a new incoming connection from this listener.
@@ -190,15 +195,22 @@
/// Creates new `TcpListener` from a `std::net::TcpListener`.
///
/// This function is intended to be used to wrap a TCP listener from the
- /// standard library in the Tokio equivalent. The conversion assumes nothing
- /// about the underlying listener; it is left up to the user to set it in
- /// non-blocking mode.
+ /// standard library in the Tokio equivalent.
///
/// This API is typically paired with the `socket2` crate and the `Socket`
/// type to build up and customize a listener before it's shipped off to the
/// backing event loop. This allows configuration of options like
/// `SO_REUSEPORT`, binding to multiple addresses, etc.
///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the listener is in
+ /// non-blocking mode. Otherwise all I/O operations on the listener
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::net::TcpListener::set_nonblocking
+ ///
/// # Examples
///
/// ```rust,no_run
@@ -216,11 +228,13 @@
///
/// # Panics
///
- /// This function panics if thread-local runtime is not set.
+ /// This function panics if it is not called from within a runtime with
+ /// IO enabled.
///
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+ #[track_caller]
pub fn from_std(listener: net::TcpListener) -> io::Result<TcpListener> {
let io = mio::net::TcpListener::from_std(listener);
let io = PollEvented::new(io)?;
@@ -267,11 +281,22 @@
.map(|io| io.into_raw_socket())
.map(|raw_socket| unsafe { std::net::TcpListener::from_raw_socket(raw_socket) })
}
+
+ #[cfg(tokio_wasi)]
+ {
+ use std::os::wasi::io::{FromRawFd, IntoRawFd};
+ self.io
+ .into_inner()
+ .map(|io| io.into_raw_fd())
+ .map(|raw_fd| unsafe { std::net::TcpListener::from_raw_fd(raw_fd) })
+ }
}
- pub(crate) fn new(listener: mio::net::TcpListener) -> io::Result<TcpListener> {
- let io = PollEvented::new(listener)?;
- Ok(TcpListener { io })
+ cfg_not_wasi! {
+ pub(crate) fn new(listener: mio::net::TcpListener) -> io::Result<TcpListener> {
+ let io = PollEvented::new(listener)?;
+ Ok(TcpListener { io })
+ }
}
/// Returns the local address that this listener is bound to.
@@ -384,6 +409,20 @@
}
}
+cfg_unstable! {
+ #[cfg(tokio_wasi)]
+ mod sys {
+ use super::TcpListener;
+ use std::os::wasi::prelude::*;
+
+ impl AsRawFd for TcpListener {
+ fn as_raw_fd(&self) -> RawFd {
+ self.io.as_raw_fd()
+ }
+ }
+ }
+}
+
#[cfg(windows)]
mod sys {
use super::TcpListener;
diff --git a/src/net/tcp/mod.rs b/src/net/tcp/mod.rs
index cb8a8b2..734eabe 100644
--- a/src/net/tcp/mod.rs
+++ b/src/net/tcp/mod.rs
@@ -2,7 +2,9 @@
pub(crate) mod listener;
-pub(crate) mod socket;
+cfg_not_wasi! {
+ pub(crate) mod socket;
+}
mod split;
pub use split::{ReadHalf, WriteHalf};
diff --git a/src/net/tcp/socket.rs b/src/net/tcp/socket.rs
index f54ff95..09349fe 100644
--- a/src/net/tcp/socket.rs
+++ b/src/net/tcp/socket.rs
@@ -8,6 +8,7 @@
use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
#[cfg(windows)]
use std::os::windows::io::{AsRawSocket, FromRawSocket, IntoRawSocket, RawSocket};
+use std::time::Duration;
cfg_net! {
/// A TCP socket that has not yet been converted to a `TcpStream` or
@@ -81,8 +82,9 @@
/// [`AsRawFd`]: https://doc.rust-lang.org/std/os/unix/io/trait.AsRawFd.html
/// [`AsRawSocket`]: https://doc.rust-lang.org/std/os/windows/io/trait.AsRawSocket.html
/// [`socket2`]: https://docs.rs/socket2/
+ #[cfg_attr(docsrs, doc(alias = "connect_std"))]
pub struct TcpSocket {
- inner: mio::net::TcpSocket,
+ inner: socket2::Socket,
}
}
@@ -117,8 +119,7 @@
/// }
/// ```
pub fn new_v4() -> io::Result<TcpSocket> {
- let inner = mio::net::TcpSocket::new_v4()?;
- Ok(TcpSocket { inner })
+ TcpSocket::new(socket2::Domain::IPV4)
}
/// Creates a new socket configured for IPv6.
@@ -151,7 +152,34 @@
/// }
/// ```
pub fn new_v6() -> io::Result<TcpSocket> {
- let inner = mio::net::TcpSocket::new_v6()?;
+ TcpSocket::new(socket2::Domain::IPV6)
+ }
+
+ fn new(domain: socket2::Domain) -> io::Result<TcpSocket> {
+ let ty = socket2::Type::STREAM;
+ #[cfg(any(
+ target_os = "android",
+ target_os = "dragonfly",
+ target_os = "freebsd",
+ target_os = "fuchsia",
+ target_os = "illumos",
+ target_os = "linux",
+ target_os = "netbsd",
+ target_os = "openbsd"
+ ))]
+ let ty = ty.nonblocking();
+ let inner = socket2::Socket::new(domain, ty, Some(socket2::Protocol::TCP))?;
+ #[cfg(not(any(
+ target_os = "android",
+ target_os = "dragonfly",
+ target_os = "freebsd",
+ target_os = "fuchsia",
+ target_os = "illumos",
+ target_os = "linux",
+ target_os = "netbsd",
+ target_os = "openbsd"
+ )))]
+ inner.set_nonblocking(true)?;
Ok(TcpSocket { inner })
}
@@ -182,7 +210,7 @@
/// }
/// ```
pub fn set_reuseaddr(&self, reuseaddr: bool) -> io::Result<()> {
- self.inner.set_reuseaddr(reuseaddr)
+ self.inner.set_reuse_address(reuseaddr)
}
/// Retrieves the value set for `SO_REUSEADDR` on this socket.
@@ -208,7 +236,7 @@
/// }
/// ```
pub fn reuseaddr(&self) -> io::Result<bool> {
- self.inner.get_reuseaddr()
+ self.inner.reuse_address()
}
/// Allows the socket to bind to an in-use port. Only available for unix systems
@@ -242,7 +270,7 @@
doc(cfg(all(unix, not(target_os = "solaris"), not(target_os = "illumos"))))
)]
pub fn set_reuseport(&self, reuseport: bool) -> io::Result<()> {
- self.inner.set_reuseport(reuseport)
+ self.inner.set_reuse_port(reuseport)
}
/// Allows the socket to bind to an in-use port. Only available for unix systems
@@ -277,14 +305,14 @@
doc(cfg(all(unix, not(target_os = "solaris"), not(target_os = "illumos"))))
)]
pub fn reuseport(&self) -> io::Result<bool> {
- self.inner.get_reuseport()
+ self.inner.reuse_port()
}
/// Sets the size of the TCP send buffer on this socket.
///
/// On most operating systems, this sets the `SO_SNDBUF` socket option.
pub fn set_send_buffer_size(&self, size: u32) -> io::Result<()> {
- self.inner.set_send_buffer_size(size)
+ self.inner.set_send_buffer_size(size as usize)
}
/// Returns the size of the TCP send buffer for this socket.
@@ -311,14 +339,14 @@
///
/// [`set_send_buffer_size`]: #method.set_send_buffer_size
pub fn send_buffer_size(&self) -> io::Result<u32> {
- self.inner.get_send_buffer_size()
+ self.inner.send_buffer_size().map(|n| n as u32)
}
/// Sets the size of the TCP receive buffer on this socket.
///
/// On most operating systems, this sets the `SO_RCVBUF` socket option.
pub fn set_recv_buffer_size(&self, size: u32) -> io::Result<()> {
- self.inner.set_recv_buffer_size(size)
+ self.inner.set_recv_buffer_size(size as usize)
}
/// Returns the size of the TCP receive buffer for this socket.
@@ -345,7 +373,112 @@
///
/// [`set_recv_buffer_size`]: #method.set_recv_buffer_size
pub fn recv_buffer_size(&self) -> io::Result<u32> {
- self.inner.get_recv_buffer_size()
+ self.inner.recv_buffer_size().map(|n| n as u32)
+ }
+
+ /// Sets the linger duration of this socket by setting the SO_LINGER option.
+ ///
+ /// This option controls the action taken when a stream has unsent messages and the stream is
+ /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the
+ /// data or until the time expires.
+ ///
+ /// If SO_LINGER is not specified, and the socket is closed, the system handles the call in a
+ /// way that allows the process to continue as quickly as possible.
+ pub fn set_linger(&self, dur: Option<Duration>) -> io::Result<()> {
+ self.inner.set_linger(dur)
+ }
+
+ /// Reads the linger duration for this socket by getting the `SO_LINGER`
+ /// option.
+ ///
+ /// For more information about this option, see [`set_linger`].
+ ///
+ /// [`set_linger`]: TcpSocket::set_linger
+ pub fn linger(&self) -> io::Result<Option<Duration>> {
+ self.inner.linger()
+ }
+
+ /// Gets the value of the `IP_TOS` option for this socket.
+ ///
+ /// For more information about this option, see [`set_tos`].
+ ///
+ /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or
+ /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
+ ///
+ /// [`set_tos`]: Self::set_tos
+ // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+ #[cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ )))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ ))))
+ )]
+ pub fn tos(&self) -> io::Result<u32> {
+ self.inner.tos()
+ }
+
+ /// Sets the value for the `IP_TOS` option on this socket.
+ ///
+ /// This value sets the type-of-service field that is used in every packet
+ /// sent from this socket.
+ ///
+ /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or
+ /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
+ // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+ #[cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ )))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ ))))
+ )]
+ pub fn set_tos(&self, tos: u32) -> io::Result<()> {
+ self.inner.set_tos(tos)
+ }
+
+ /// Gets the value for the `SO_BINDTODEVICE` option on this socket
+ ///
+ /// This value gets the socket binded device's interface name.
+ #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",)))
+ )]
+ pub fn device(&self) -> io::Result<Option<Vec<u8>>> {
+ self.inner.device()
+ }
+
+ /// Sets the value for the `SO_BINDTODEVICE` option on this socket
+ ///
+ /// If a socket is bound to an interface, only packets received from that
+ /// particular interface are processed by the socket. Note that this only
+ /// works for some socket types, particularly `AF_INET` sockets.
+ ///
+ /// If `interface` is `None` or an empty string it removes the binding.
+ #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))))
+ )]
+ pub fn bind_device(&self, interface: Option<&[u8]>) -> io::Result<()> {
+ self.inner.bind_device(interface)
}
/// Gets the local address of this socket.
@@ -371,7 +504,12 @@
/// }
/// ```
pub fn local_addr(&self) -> io::Result<SocketAddr> {
- self.inner.get_localaddr()
+ self.inner.local_addr().and_then(convert_address)
+ }
+
+ /// Returns the value of the `SO_ERROR` option.
+ pub fn take_error(&self) -> io::Result<Option<io::Error>> {
+ self.inner.take_error()
}
/// Binds the socket to the given address.
@@ -403,7 +541,7 @@
/// }
/// ```
pub fn bind(&self, addr: SocketAddr) -> io::Result<()> {
- self.inner.bind(addr)
+ self.inner.bind(&addr.into())
}
/// Establishes a TCP connection with a peer at the specified socket address.
@@ -439,7 +577,32 @@
/// }
/// ```
pub async fn connect(self, addr: SocketAddr) -> io::Result<TcpStream> {
- let mio = self.inner.connect(addr)?;
+ if let Err(err) = self.inner.connect(&addr.into()) {
+ #[cfg(unix)]
+ if err.raw_os_error() != Some(libc::EINPROGRESS) {
+ return Err(err);
+ }
+ #[cfg(windows)]
+ if err.kind() != io::ErrorKind::WouldBlock {
+ return Err(err);
+ }
+ }
+ #[cfg(unix)]
+ let mio = {
+ use std::os::unix::io::{FromRawFd, IntoRawFd};
+
+ let raw_fd = self.inner.into_raw_fd();
+ unsafe { mio::net::TcpStream::from_raw_fd(raw_fd) }
+ };
+
+ #[cfg(windows)]
+ let mio = {
+ use std::os::windows::io::{FromRawSocket, IntoRawSocket};
+
+ let raw_socket = self.inner.into_raw_socket();
+ unsafe { mio::net::TcpStream::from_raw_socket(raw_socket) }
+ };
+
TcpStream::connect_mio(mio).await
}
@@ -479,7 +642,23 @@
/// }
/// ```
pub fn listen(self, backlog: u32) -> io::Result<TcpListener> {
- let mio = self.inner.listen(backlog)?;
+ self.inner.listen(backlog as i32)?;
+ #[cfg(unix)]
+ let mio = {
+ use std::os::unix::io::{FromRawFd, IntoRawFd};
+
+ let raw_fd = self.inner.into_raw_fd();
+ unsafe { mio::net::TcpListener::from_raw_fd(raw_fd) }
+ };
+
+ #[cfg(windows)]
+ let mio = {
+ use std::os::windows::io::{FromRawSocket, IntoRawSocket};
+
+ let raw_socket = self.inner.into_raw_socket();
+ unsafe { mio::net::TcpListener::from_raw_socket(raw_socket) }
+ };
+
TcpListener::new(mio)
}
@@ -491,6 +670,15 @@
/// [`std::net::TcpStream`]: struct@std::net::TcpStream
/// [`socket2`]: https://docs.rs/socket2/
///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the socket is in
+ /// non-blocking mode. Otherwise all I/O operations on the socket
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::net::TcpStream::set_nonblocking
+ ///
/// # Examples
///
/// ```
@@ -499,8 +687,8 @@
///
/// #[tokio::main]
/// async fn main() -> std::io::Result<()> {
- ///
/// let socket2_socket = Socket::new(Domain::IPV4, Type::STREAM, None)?;
+ /// socket2_socket.set_nonblocking(true)?;
///
/// let socket = TcpSocket::from_std_stream(socket2_socket.into());
///
@@ -526,6 +714,16 @@
}
}
+fn convert_address(address: socket2::SockAddr) -> io::Result<SocketAddr> {
+ match address.as_socket() {
+ Some(address) => Ok(address),
+ None => Err(io::Error::new(
+ io::ErrorKind::InvalidInput,
+ "invalid address family (not IPv4 or IPv6)",
+ )),
+ }
+}
+
impl fmt::Debug for TcpSocket {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
self.inner.fmt(fmt)
@@ -548,7 +746,7 @@
/// The caller is responsible for ensuring that the socket is in
/// non-blocking mode.
unsafe fn from_raw_fd(fd: RawFd) -> TcpSocket {
- let inner = mio::net::TcpSocket::from_raw_fd(fd);
+ let inner = socket2::Socket::from_raw_fd(fd);
TcpSocket { inner }
}
}
@@ -583,7 +781,7 @@
/// The caller is responsible for ensuring that the socket is in
/// non-blocking mode.
unsafe fn from_raw_socket(socket: RawSocket) -> TcpSocket {
- let inner = mio::net::TcpSocket::from_raw_socket(socket);
+ let inner = socket2::Socket::from_raw_socket(socket);
TcpSocket { inner }
}
}
diff --git a/src/net/tcp/split.rs b/src/net/tcp/split.rs
index 0e02928..343d4fd 100644
--- a/src/net/tcp/split.rs
+++ b/src/net/tcp/split.rs
@@ -145,6 +145,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// This function is equivalent to [`TcpStream::ready`].
///
/// # Cancel safety
@@ -190,8 +196,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
- /// and will no longer yield data. If the stream is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The stream's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the stream is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> {
self.0.try_read(buf)
@@ -269,6 +279,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// This function is equivalent to [`TcpStream::ready`].
///
/// # Cancel safety
diff --git a/src/net/tcp/split_owned.rs b/src/net/tcp/split_owned.rs
index ef4e7b5..53fc5f0 100644
--- a/src/net/tcp/split_owned.rs
+++ b/src/net/tcp/split_owned.rs
@@ -200,6 +200,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// This function is equivalent to [`TcpStream::ready`].
///
/// # Cancel safety
@@ -245,8 +251,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
- /// and will no longer yield data. If the stream is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The stream's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the stream is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> {
self.inner.try_read(buf)
@@ -351,6 +361,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// This function is equivalent to [`TcpStream::ready`].
///
/// # Cancel safety
@@ -474,12 +490,12 @@
impl AsRef<TcpStream> for OwnedReadHalf {
fn as_ref(&self) -> &TcpStream {
- &*self.inner
+ &self.inner
}
}
impl AsRef<TcpStream> for OwnedWriteHalf {
fn as_ref(&self) -> &TcpStream {
- &*self.inner
+ &self.inner
}
}
diff --git a/src/net/tcp/stream.rs b/src/net/tcp/stream.rs
index 60d20fd..b17d33f 100644
--- a/src/net/tcp/stream.rs
+++ b/src/net/tcp/stream.rs
@@ -1,8 +1,12 @@
-use crate::future::poll_fn;
+cfg_not_wasi! {
+ use crate::future::poll_fn;
+ use crate::net::{to_socket_addrs, ToSocketAddrs};
+ use std::time::Duration;
+}
+
use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready};
use crate::net::tcp::split::{split, ReadHalf, WriteHalf};
use crate::net::tcp::split_owned::{split_owned, OwnedReadHalf, OwnedWriteHalf};
-use crate::net::{to_socket_addrs, ToSocketAddrs};
use std::convert::TryFrom;
use std::fmt;
@@ -10,7 +14,6 @@
use std::net::{Shutdown, SocketAddr};
use std::pin::Pin;
use std::task::{Context, Poll};
-use std::time::Duration;
cfg_io_util! {
use bytes::BufMut;
@@ -70,86 +73,88 @@
}
impl TcpStream {
- /// Opens a TCP connection to a remote host.
- ///
- /// `addr` is an address of the remote host. Anything which implements the
- /// [`ToSocketAddrs`] trait can be supplied as the address. If `addr`
- /// yields multiple addresses, connect will be attempted with each of the
- /// addresses until a connection is successful. If none of the addresses
- /// result in a successful connection, the error returned from the last
- /// connection attempt (the last address) is returned.
- ///
- /// To configure the socket before connecting, you can use the [`TcpSocket`]
- /// type.
- ///
- /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs
- /// [`TcpSocket`]: struct@crate::net::TcpSocket
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use tokio::net::TcpStream;
- /// use tokio::io::AsyncWriteExt;
- /// use std::error::Error;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn Error>> {
- /// // Connect to a peer
- /// let mut stream = TcpStream::connect("127.0.0.1:8080").await?;
- ///
- /// // Write some data.
- /// stream.write_all(b"hello world!").await?;
- ///
- /// Ok(())
- /// }
- /// ```
- ///
- /// The [`write_all`] method is defined on the [`AsyncWriteExt`] trait.
- ///
- /// [`write_all`]: fn@crate::io::AsyncWriteExt::write_all
- /// [`AsyncWriteExt`]: trait@crate::io::AsyncWriteExt
- pub async fn connect<A: ToSocketAddrs>(addr: A) -> io::Result<TcpStream> {
- let addrs = to_socket_addrs(addr).await?;
+ cfg_not_wasi! {
+ /// Opens a TCP connection to a remote host.
+ ///
+ /// `addr` is an address of the remote host. Anything which implements the
+ /// [`ToSocketAddrs`] trait can be supplied as the address. If `addr`
+ /// yields multiple addresses, connect will be attempted with each of the
+ /// addresses until a connection is successful. If none of the addresses
+ /// result in a successful connection, the error returned from the last
+ /// connection attempt (the last address) is returned.
+ ///
+ /// To configure the socket before connecting, you can use the [`TcpSocket`]
+ /// type.
+ ///
+ /// [`ToSocketAddrs`]: trait@crate::net::ToSocketAddrs
+ /// [`TcpSocket`]: struct@crate::net::TcpSocket
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::TcpStream;
+ /// use tokio::io::AsyncWriteExt;
+ /// use std::error::Error;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn Error>> {
+ /// // Connect to a peer
+ /// let mut stream = TcpStream::connect("127.0.0.1:8080").await?;
+ ///
+ /// // Write some data.
+ /// stream.write_all(b"hello world!").await?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ ///
+ /// The [`write_all`] method is defined on the [`AsyncWriteExt`] trait.
+ ///
+ /// [`write_all`]: fn@crate::io::AsyncWriteExt::write_all
+ /// [`AsyncWriteExt`]: trait@crate::io::AsyncWriteExt
+ pub async fn connect<A: ToSocketAddrs>(addr: A) -> io::Result<TcpStream> {
+ let addrs = to_socket_addrs(addr).await?;
- let mut last_err = None;
+ let mut last_err = None;
- for addr in addrs {
- match TcpStream::connect_addr(addr).await {
- Ok(stream) => return Ok(stream),
- Err(e) => last_err = Some(e),
+ for addr in addrs {
+ match TcpStream::connect_addr(addr).await {
+ Ok(stream) => return Ok(stream),
+ Err(e) => last_err = Some(e),
+ }
}
+
+ Err(last_err.unwrap_or_else(|| {
+ io::Error::new(
+ io::ErrorKind::InvalidInput,
+ "could not resolve to any address",
+ )
+ }))
}
- Err(last_err.unwrap_or_else(|| {
- io::Error::new(
- io::ErrorKind::InvalidInput,
- "could not resolve to any address",
- )
- }))
- }
-
- /// Establishes a connection to the specified `addr`.
- async fn connect_addr(addr: SocketAddr) -> io::Result<TcpStream> {
- let sys = mio::net::TcpStream::connect(addr)?;
- TcpStream::connect_mio(sys).await
- }
-
- pub(crate) async fn connect_mio(sys: mio::net::TcpStream) -> io::Result<TcpStream> {
- let stream = TcpStream::new(sys)?;
-
- // Once we've connected, wait for the stream to be writable as
- // that's when the actual connection has been initiated. Once we're
- // writable we check for `take_socket_error` to see if the connect
- // actually hit an error or not.
- //
- // If all that succeeded then we ship everything on up.
- poll_fn(|cx| stream.io.registration().poll_write_ready(cx)).await?;
-
- if let Some(e) = stream.io.take_error()? {
- return Err(e);
+ /// Establishes a connection to the specified `addr`.
+ async fn connect_addr(addr: SocketAddr) -> io::Result<TcpStream> {
+ let sys = mio::net::TcpStream::connect(addr)?;
+ TcpStream::connect_mio(sys).await
}
- Ok(stream)
+ pub(crate) async fn connect_mio(sys: mio::net::TcpStream) -> io::Result<TcpStream> {
+ let stream = TcpStream::new(sys)?;
+
+ // Once we've connected, wait for the stream to be writable as
+ // that's when the actual connection has been initiated. Once we're
+ // writable we check for `take_socket_error` to see if the connect
+ // actually hit an error or not.
+ //
+ // If all that succeeded then we ship everything on up.
+ poll_fn(|cx| stream.io.registration().poll_write_ready(cx)).await?;
+
+ if let Some(e) = stream.io.take_error()? {
+ return Err(e);
+ }
+
+ Ok(stream)
+ }
}
pub(crate) fn new(connected: mio::net::TcpStream) -> io::Result<TcpStream> {
@@ -160,9 +165,16 @@
/// Creates new `TcpStream` from a `std::net::TcpStream`.
///
/// This function is intended to be used to wrap a TCP stream from the
- /// standard library in the Tokio equivalent. The conversion assumes nothing
- /// about the underlying stream; it is left up to the user to set it in
- /// non-blocking mode.
+ /// standard library in the Tokio equivalent.
+ ///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the stream is in
+ /// non-blocking mode. Otherwise all I/O operations on the stream
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::net::TcpStream::set_nonblocking
///
/// # Examples
///
@@ -181,11 +193,13 @@
///
/// # Panics
///
- /// This function panics if thread-local runtime is not set.
+ /// This function panics if it is not called from within a runtime with
+ /// IO enabled.
///
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+ #[track_caller]
pub fn from_std(stream: std::net::TcpStream) -> io::Result<TcpStream> {
let io = mio::net::TcpStream::from_std(stream);
let io = PollEvented::new(io)?;
@@ -244,6 +258,15 @@
.map(|io| io.into_raw_socket())
.map(|raw_socket| unsafe { std::net::TcpStream::from_raw_socket(raw_socket) })
}
+
+ #[cfg(tokio_wasi)]
+ {
+ use std::os::wasi::io::{FromRawFd, IntoRawFd};
+ self.io
+ .into_inner()
+ .map(|io| io.into_raw_fd())
+ .map(|raw_fd| unsafe { std::net::TcpStream::from_raw_fd(raw_fd) })
+ }
}
/// Returns the local address that this stream is bound to.
@@ -264,6 +287,11 @@
self.io.local_addr()
}
+ /// Returns the value of the `SO_ERROR` option.
+ pub fn take_error(&self) -> io::Result<Option<io::Error>> {
+ self.io.take_error()
+ }
+
/// Returns the remote address that this stream is connected to.
///
/// # Examples
@@ -356,6 +384,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Cancel safety
///
/// This method is cancel safe. Once a readiness event occurs, the method
@@ -387,7 +421,7 @@
/// // if the readiness event is a false positive.
/// match stream.try_read(&mut data) {
/// Ok(n) => {
- /// println!("read {} bytes", n);
+ /// println!("read {} bytes", n);
/// }
/// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
/// continue;
@@ -526,8 +560,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
- /// and will no longer yield data. If the stream is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The stream's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the stream is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
///
/// # Examples
@@ -939,7 +977,7 @@
/// Tries to read or write from the socket using a user-provided IO operation.
///
/// If the socket is ready, the provided closure is called. The closure
- /// should attempt to perform IO operation from the socket by manually
+ /// should attempt to perform IO operation on the socket by manually
/// calling the appropriate syscall. If the operation fails because the
/// socket is not actually ready, then the closure should return a
/// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -958,6 +996,11 @@
/// defined on the Tokio `TcpStream` type, as this will mess with the
/// readiness flag and can cause the socket to behave incorrectly.
///
+ /// This method is not intended to be used with combined interests.
+ /// The closure should perform only one type of IO operation, so it should not
+ /// require more than one ready state. This method may panic or sleep forever
+ /// if it is called with a combined interest.
+ ///
/// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
///
/// [`readable()`]: TcpStream::readable()
@@ -968,7 +1011,9 @@
interest: Interest,
f: impl FnOnce() -> io::Result<R>,
) -> io::Result<R> {
- self.io.registration().try_io(interest, f)
+ self.io
+ .registration()
+ .try_io(interest, || self.io.try_io(f))
}
/// Receives data on the socket from the remote address to which it is
@@ -1070,69 +1115,53 @@
self.io.set_nodelay(nodelay)
}
- /// Reads the linger duration for this socket by getting the `SO_LINGER`
- /// option.
- ///
- /// For more information about this option, see [`set_linger`].
- ///
- /// [`set_linger`]: TcpStream::set_linger
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use tokio::net::TcpStream;
- ///
- /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
- /// let stream = TcpStream::connect("127.0.0.1:8080").await?;
- ///
- /// println!("{:?}", stream.linger()?);
- /// # Ok(())
- /// # }
- /// ```
- pub fn linger(&self) -> io::Result<Option<Duration>> {
- let mio_socket = std::mem::ManuallyDrop::new(self.to_mio());
-
- mio_socket.get_linger()
- }
-
- /// Sets the linger duration of this socket by setting the SO_LINGER option.
- ///
- /// This option controls the action taken when a stream has unsent messages and the stream is
- /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the
- /// data or until the time expires.
- ///
- /// If SO_LINGER is not specified, and the stream is closed, the system handles the call in a
- /// way that allows the process to continue as quickly as possible.
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use tokio::net::TcpStream;
- ///
- /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
- /// let stream = TcpStream::connect("127.0.0.1:8080").await?;
- ///
- /// stream.set_linger(None)?;
- /// # Ok(())
- /// # }
- /// ```
- pub fn set_linger(&self, dur: Option<Duration>) -> io::Result<()> {
- let mio_socket = std::mem::ManuallyDrop::new(self.to_mio());
-
- mio_socket.set_linger(dur)
- }
-
- fn to_mio(&self) -> mio::net::TcpSocket {
- #[cfg(windows)]
- {
- use std::os::windows::io::{AsRawSocket, FromRawSocket};
- unsafe { mio::net::TcpSocket::from_raw_socket(self.as_raw_socket()) }
+ cfg_not_wasi! {
+ /// Reads the linger duration for this socket by getting the `SO_LINGER`
+ /// option.
+ ///
+ /// For more information about this option, see [`set_linger`].
+ ///
+ /// [`set_linger`]: TcpStream::set_linger
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::TcpStream;
+ ///
+ /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
+ /// let stream = TcpStream::connect("127.0.0.1:8080").await?;
+ ///
+ /// println!("{:?}", stream.linger()?);
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn linger(&self) -> io::Result<Option<Duration>> {
+ socket2::SockRef::from(self).linger()
}
- #[cfg(unix)]
- {
- use std::os::unix::io::{AsRawFd, FromRawFd};
- unsafe { mio::net::TcpSocket::from_raw_fd(self.as_raw_fd()) }
+ /// Sets the linger duration of this socket by setting the SO_LINGER option.
+ ///
+ /// This option controls the action taken when a stream has unsent messages and the stream is
+ /// closed. If SO_LINGER is set, the system shall block the process until it can transmit the
+ /// data or until the time expires.
+ ///
+ /// If SO_LINGER is not specified, and the stream is closed, the system handles the call in a
+ /// way that allows the process to continue as quickly as possible.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::TcpStream;
+ ///
+ /// # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
+ /// let stream = TcpStream::connect("127.0.0.1:8080").await?;
+ ///
+ /// stream.set_linger(None)?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn set_linger(&self, dur: Option<Duration>) -> io::Result<()> {
+ socket2::SockRef::from(self).set_linger(dur)
}
}
@@ -1326,3 +1355,15 @@
}
}
}
+
+#[cfg(all(tokio_unstable, tokio_wasi))]
+mod sys {
+ use super::TcpStream;
+ use std::os::wasi::prelude::*;
+
+ impl AsRawFd for TcpStream {
+ fn as_raw_fd(&self) -> RawFd {
+ self.io.as_raw_fd()
+ }
+ }
+}
diff --git a/src/net/udp.rs b/src/net/udp.rs
index 504d74e..213d914 100644
--- a/src/net/udp.rs
+++ b/src/net/udp.rs
@@ -128,6 +128,10 @@
/// This function will create a new UDP socket and attempt to bind it to
/// the `addr` provided.
///
+ /// Binding with a port number of 0 will request that the OS assigns a port
+ /// to this listener. The port allocated can be queried via the `local_addr`
+ /// method.
+ ///
/// # Example
///
/// ```no_run
@@ -166,6 +170,7 @@
UdpSocket::new(sys)
}
+ #[track_caller]
fn new(socket: mio::net::UdpSocket) -> io::Result<UdpSocket> {
let io = PollEvented::new(socket)?;
Ok(UdpSocket { io })
@@ -174,14 +179,21 @@
/// Creates new `UdpSocket` from a previously bound `std::net::UdpSocket`.
///
/// This function is intended to be used to wrap a UDP socket from the
- /// standard library in the Tokio equivalent. The conversion assumes nothing
- /// about the underlying socket; it is left up to the user to set it in
- /// non-blocking mode.
+ /// standard library in the Tokio equivalent.
///
/// This can be used in conjunction with socket2's `Socket` interface to
/// configure a socket before it's handed off, such as setting options like
/// `reuse_address` or binding to multiple addresses.
///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the socket is in
+ /// non-blocking mode. Otherwise all I/O operations on the socket
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::net::UdpSocket::set_nonblocking
+ ///
/// # Panics
///
/// This function panics if thread-local runtime is not set.
@@ -206,6 +218,7 @@
/// # Ok(())
/// # }
/// ```
+ #[track_caller]
pub fn from_std(socket: net::UdpSocket) -> io::Result<UdpSocket> {
let io = mio::net::UdpSocket::from_std(socket);
UdpSocket::new(io)
@@ -253,6 +266,10 @@
}
}
+ fn as_socket(&self) -> socket2::SockRef<'_> {
+ socket2::SockRef::from(self)
+ }
+
/// Returns the local address that this socket is bound to.
///
/// # Example
@@ -274,6 +291,28 @@
self.io.local_addr()
}
+ /// Returns the socket address of the remote peer this socket was connected to.
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// use tokio::net::UdpSocket;
+ ///
+ /// # use std::{io, net::SocketAddr};
+ /// # #[tokio::main]
+ /// # async fn main() -> io::Result<()> {
+ /// let addr = "0.0.0.0:8080".parse::<SocketAddr>().unwrap();
+ /// let peer = "127.0.0.1:11100".parse::<SocketAddr>().unwrap();
+ /// let sock = UdpSocket::bind(addr).await?;
+ /// sock.connect(peer).await?;
+ /// assert_eq!(peer, sock.peer_addr()?);
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn peer_addr(&self) -> io::Result<SocketAddr> {
+ self.io.peer_addr()
+ }
+
/// Connects the UDP socket setting the default destination for send() and
/// limiting packets that are read via recv from the address specified in
/// `addr`.
@@ -325,7 +364,9 @@
///
/// The function may complete without the socket being ready. This is a
/// false-positive and attempting an operation will return with
- /// `io::ErrorKind::WouldBlock`.
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
///
/// # Cancel safety
///
@@ -708,7 +749,7 @@
///
/// # Cancel safety
///
- /// This method is cancel safe. If `recv_from` is used as the event in a
+ /// This method is cancel safe. If `recv` is used as the event in a
/// [`tokio::select!`](crate::select) statement and some other branch
/// completes first, it is guaranteed that no messages were received on this
/// socket.
@@ -893,7 +934,7 @@
// Safety: We trust `UdpSocket::recv` to have filled up `n` bytes in the
// buffer.
- let n = (&*self.io).recv(dst)?;
+ let n = (*self.io).recv(dst)?;
unsafe {
buf.advance_mut(n);
@@ -957,7 +998,7 @@
// Safety: We trust `UdpSocket::recv_from` to have filled up `n` bytes in the
// buffer.
- let (n, addr) = (&*self.io).recv_from(dst)?;
+ let (n, addr) = (*self.io).recv_from(dst)?;
unsafe {
buf.advance_mut(n);
@@ -1239,7 +1280,7 @@
/// Tries to read or write from the socket using a user-provided IO operation.
///
/// If the socket is ready, the provided closure is called. The closure
- /// should attempt to perform IO operation from the socket by manually
+ /// should attempt to perform IO operation on the socket by manually
/// calling the appropriate syscall. If the operation fails because the
/// socket is not actually ready, then the closure should return a
/// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -1258,6 +1299,11 @@
/// defined on the Tokio `UdpSocket` type, as this will mess with the
/// readiness flag and can cause the socket to behave incorrectly.
///
+ /// This method is not intended to be used with combined interests.
+ /// The closure should perform only one type of IO operation, so it should not
+ /// require more than one ready state. This method may panic or sleep forever
+ /// if it is called with a combined interest.
+ ///
/// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
///
/// [`readable()`]: UdpSocket::readable()
@@ -1268,7 +1314,9 @@
interest: Interest,
f: impl FnOnce() -> io::Result<R>,
) -> io::Result<R> {
- self.io.registration().try_io(interest, f)
+ self.io
+ .registration()
+ .try_io(interest, || self.io.try_io(f))
}
/// Receives data from the socket, without removing it from the input queue.
@@ -1480,6 +1528,89 @@
self.io.set_ttl(ttl)
}
+ /// Gets the value of the `IP_TOS` option for this socket.
+ ///
+ /// For more information about this option, see [`set_tos`].
+ ///
+ /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or
+ /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
+ ///
+ /// [`set_tos`]: Self::set_tos
+ // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+ #[cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ )))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ ))))
+ )]
+ pub fn tos(&self) -> io::Result<u32> {
+ self.as_socket().tos()
+ }
+
+ /// Sets the value for the `IP_TOS` option on this socket.
+ ///
+ /// This value sets the type-of-service field that is used in every packet
+ /// sent from this socket.
+ ///
+ /// **NOTE:** On Windows, `IP_TOS` is only supported on [Windows 8+ or
+ /// Windows Server 2012+.](https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options)
+ // https://docs.rs/socket2/0.4.2/src/socket2/socket.rs.html#1178
+ #[cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ )))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(not(any(
+ target_os = "fuchsia",
+ target_os = "redox",
+ target_os = "solaris",
+ target_os = "illumos",
+ ))))
+ )]
+ pub fn set_tos(&self, tos: u32) -> io::Result<()> {
+ self.as_socket().set_tos(tos)
+ }
+
+ /// Gets the value for the `SO_BINDTODEVICE` option on this socket
+ ///
+ /// This value gets the socket-bound device's interface name.
+ #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux",)))
+ )]
+ pub fn device(&self) -> io::Result<Option<Vec<u8>>> {
+ self.as_socket().device()
+ }
+
+ /// Sets the value for the `SO_BINDTODEVICE` option on this socket
+ ///
+ /// If a socket is bound to an interface, only packets received from that
+ /// particular interface are processed by the socket. Note that this only
+ /// works for some socket types, particularly `AF_INET` sockets.
+ ///
+ /// If `interface` is `None` or an empty string it removes the binding.
+ #[cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))]
+ #[cfg_attr(
+ docsrs,
+ doc(cfg(all(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))))
+ )]
+ pub fn bind_device(&self, interface: Option<&[u8]>) -> io::Result<()> {
+ self.as_socket().bind_device(interface)
+ }
+
/// Executes an operation of the `IP_ADD_MEMBERSHIP` type.
///
/// This function specifies a new multicast group for this socket to join.
diff --git a/src/net/unix/datagram/socket.rs b/src/net/unix/datagram/socket.rs
index d5b6186..76c6b19 100644
--- a/src/net/unix/datagram/socket.rs
+++ b/src/net/unix/datagram/socket.rs
@@ -104,7 +104,9 @@
///
/// The function may complete without the socket being ready. This is a
/// false-positive and attempting an operation will return with
- /// `io::ErrorKind::WouldBlock`.
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
///
/// # Cancel safety
///
@@ -424,13 +426,21 @@
/// Creates new `UnixDatagram` from a `std::os::unix::net::UnixDatagram`.
///
/// This function is intended to be used to wrap a UnixDatagram from the
- /// standard library in the Tokio equivalent. The conversion assumes
- /// nothing about the underlying datagram; it is left up to the user to set
- /// it in non-blocking mode.
+ /// standard library in the Tokio equivalent.
+ ///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the socker is in
+ /// non-blocking mode. Otherwise all I/O operations on the socket
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::os::unix::net::UnixDatagram::set_nonblocking
///
/// # Panics
///
- /// This function panics if thread-local runtime is not set.
+ /// This function panics if it is not called from within a runtime with
+ /// IO enabled.
///
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a Tokio runtime, otherwise runtime can be set
@@ -457,6 +467,7 @@
/// # Ok(())
/// # }
/// ```
+ #[track_caller]
pub fn from_std(datagram: net::UnixDatagram) -> io::Result<UnixDatagram> {
let socket = mio::net::UnixDatagram::from_std(datagram);
let io = PollEvented::new(socket)?;
@@ -466,21 +477,19 @@
/// Turns a [`tokio::net::UnixDatagram`] into a [`std::os::unix::net::UnixDatagram`].
///
/// The returned [`std::os::unix::net::UnixDatagram`] will have nonblocking
- /// mode set as `true`. Use [`set_nonblocking`] to change the blocking mode
+ /// mode set as `true`. Use [`set_nonblocking`] to change the blocking mode
/// if needed.
///
/// # Examples
///
/// ```rust,no_run
- /// use std::error::Error;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn Error>> {
- /// let tokio_socket = tokio::net::UnixDatagram::bind("127.0.0.1:0")?;
- /// let std_socket = tokio_socket.into_std()?;
- /// std_socket.set_nonblocking(false)?;
- /// Ok(())
- /// }
+ /// # use std::error::Error;
+ /// # async fn dox() -> Result<(), Box<dyn Error>> {
+ /// let tokio_socket = tokio::net::UnixDatagram::bind("/path/to/the/socket")?;
+ /// let std_socket = tokio_socket.into_std()?;
+ /// std_socket.set_nonblocking(false)?;
+ /// # Ok(())
+ /// # }
/// ```
///
/// [`tokio::net::UnixDatagram`]: UnixDatagram
@@ -844,7 +853,7 @@
// Safety: We trust `UnixDatagram::recv_from` to have filled up `n` bytes in the
// buffer.
- let (n, addr) = (&*self.io).recv_from(dst)?;
+ let (n, addr) = (*self.io).recv_from(dst)?;
unsafe {
buf.advance_mut(n);
@@ -907,7 +916,7 @@
// Safety: We trust `UnixDatagram::recv` to have filled up `n` bytes in the
// buffer.
- let n = (&*self.io).recv(dst)?;
+ let n = (*self.io).recv(dst)?;
unsafe {
buf.advance_mut(n);
@@ -1212,7 +1221,7 @@
/// Tries to read or write from the socket using a user-provided IO operation.
///
/// If the socket is ready, the provided closure is called. The closure
- /// should attempt to perform IO operation from the socket by manually
+ /// should attempt to perform IO operation on the socket by manually
/// calling the appropriate syscall. If the operation fails because the
/// socket is not actually ready, then the closure should return a
/// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -1231,6 +1240,11 @@
/// defined on the Tokio `UnixDatagram` type, as this will mess with the
/// readiness flag and can cause the socket to behave incorrectly.
///
+ /// This method is not intended to be used with combined interests.
+ /// The closure should perform only one type of IO operation, so it should not
+ /// require more than one ready state. This method may panic or sleep forever
+ /// if it is called with a combined interest.
+ ///
/// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
///
/// [`readable()`]: UnixDatagram::readable()
@@ -1241,7 +1255,9 @@
interest: Interest,
f: impl FnOnce() -> io::Result<R>,
) -> io::Result<R> {
- self.io.registration().try_io(interest, f)
+ self.io
+ .registration()
+ .try_io(interest, || self.io.try_io(f))
}
/// Returns the local address that this socket is bound to.
diff --git a/src/net/unix/listener.rs b/src/net/unix/listener.rs
index 1785f8b..9887f73 100644
--- a/src/net/unix/listener.rs
+++ b/src/net/unix/listener.rs
@@ -54,11 +54,13 @@
///
/// # Panics
///
- /// This function panics if thread-local runtime is not set.
+ /// This function panics if it is not called from within a runtime with
+ /// IO enabled.
///
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+ #[track_caller]
pub fn bind<P>(path: P) -> io::Result<UnixListener>
where
P: AsRef<Path>,
@@ -71,17 +73,41 @@
/// Creates new `UnixListener` from a `std::os::unix::net::UnixListener `.
///
/// This function is intended to be used to wrap a UnixListener from the
- /// standard library in the Tokio equivalent. The conversion assumes
- /// nothing about the underlying listener; it is left up to the user to set
- /// it in non-blocking mode.
+ /// standard library in the Tokio equivalent.
+ ///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the listener is in
+ /// non-blocking mode. Otherwise all I/O operations on the listener
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::os::unix::net::UnixListener::set_nonblocking
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::UnixListener;
+ /// use std::os::unix::net::UnixListener as StdUnixListener;
+ /// # use std::error::Error;
+ ///
+ /// # async fn dox() -> Result<(), Box<dyn Error>> {
+ /// let std_listener = StdUnixListener::bind("/path/to/the/socket")?;
+ /// std_listener.set_nonblocking(true)?;
+ /// let listener = UnixListener::from_std(std_listener)?;
+ /// # Ok(())
+ /// # }
+ /// ```
///
/// # Panics
///
- /// This function panics if thread-local runtime is not set.
+ /// This function panics if it is not called from within a runtime with
+ /// IO enabled.
///
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+ #[track_caller]
pub fn from_std(listener: net::UnixListener) -> io::Result<UnixListener> {
let listener = mio::net::UnixListener::from_std(listener);
let io = PollEvented::new(listener)?;
@@ -91,20 +117,18 @@
/// Turns a [`tokio::net::UnixListener`] into a [`std::os::unix::net::UnixListener`].
///
/// The returned [`std::os::unix::net::UnixListener`] will have nonblocking mode
- /// set as `true`. Use [`set_nonblocking`] to change the blocking mode if needed.
+ /// set as `true`. Use [`set_nonblocking`] to change the blocking mode if needed.
///
/// # Examples
///
/// ```rust,no_run
- /// use std::error::Error;
- ///
- /// #[tokio::main]
- /// async fn main() -> Result<(), Box<dyn Error>> {
- /// let tokio_listener = tokio::net::UnixListener::bind("127.0.0.1:0")?;
- /// let std_listener = tokio_listener.into_std()?;
- /// std_listener.set_nonblocking(false)?;
- /// Ok(())
- /// }
+ /// # use std::error::Error;
+ /// # async fn dox() -> Result<(), Box<dyn Error>> {
+ /// let tokio_listener = tokio::net::UnixListener::bind("/path/to/the/socket")?;
+ /// let std_listener = tokio_listener.into_std()?;
+ /// std_listener.set_nonblocking(false)?;
+ /// # Ok(())
+ /// # }
/// ```
///
/// [`tokio::net::UnixListener`]: UnixListener
diff --git a/src/net/unix/mod.rs b/src/net/unix/mod.rs
index 14cb456..97b6327 100644
--- a/src/net/unix/mod.rs
+++ b/src/net/unix/mod.rs
@@ -1,5 +1,4 @@
//! Unix domain socket utility types.
-
// This module does not currently provide any public API, but it was
// unintentionally defined as a public module. Hide it from the documentation
// instead of changing it to a private module to avoid breakage.
@@ -22,3 +21,15 @@
mod ucred;
pub use ucred::UCred;
+
+/// A type representing process and process group IDs.
+#[allow(non_camel_case_types)]
+pub type uid_t = u32;
+
+/// A type representing user ID.
+#[allow(non_camel_case_types)]
+pub type gid_t = u32;
+
+/// A type representing group ID.
+#[allow(non_camel_case_types)]
+pub type pid_t = i32;
diff --git a/src/net/unix/split.rs b/src/net/unix/split.rs
index d4686c2..816a257 100644
--- a/src/net/unix/split.rs
+++ b/src/net/unix/split.rs
@@ -100,8 +100,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
- /// and will no longer yield data. If the stream is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The stream's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the stream is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> {
self.0.try_read(buf)
@@ -178,6 +182,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Cancel safety
///
/// This method is cancel safe. Once a readiness event occurs, the method
diff --git a/src/net/unix/split_owned.rs b/src/net/unix/split_owned.rs
index 9c3a2a4..2cb561d 100644
--- a/src/net/unix/split_owned.rs
+++ b/src/net/unix/split_owned.rs
@@ -114,6 +114,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Cancel safety
///
/// This method is cancel safe. Once a readiness event occurs, the method
@@ -155,8 +161,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
- /// and will no longer yield data. If the stream is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The stream's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the stream is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
pub fn try_read(&self, buf: &mut [u8]) -> io::Result<usize> {
self.inner.try_read(buf)
@@ -261,6 +271,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Cancel safety
///
/// This method is cancel safe. Once a readiness event occurs, the method
@@ -382,12 +398,12 @@
impl AsRef<UnixStream> for OwnedReadHalf {
fn as_ref(&self) -> &UnixStream {
- &*self.inner
+ &self.inner
}
}
impl AsRef<UnixStream> for OwnedWriteHalf {
fn as_ref(&self) -> &UnixStream {
- &*self.inner
+ &self.inner
}
}
diff --git a/src/net/unix/stream.rs b/src/net/unix/stream.rs
index 4e7ef87..c249bf4 100644
--- a/src/net/unix/stream.rs
+++ b/src/net/unix/stream.rs
@@ -22,8 +22,8 @@
cfg_net_unix! {
/// A structure representing a connected Unix socket.
///
- /// This socket can be connected directly with `UnixStream::connect` or accepted
- /// from a listener with `UnixListener::incoming`. Additionally, a pair of
+ /// This socket can be connected directly with [`UnixStream::connect`] or accepted
+ /// from a listener with [`UnixListener::accept`]. Additionally, a pair of
/// anonymous Unix sockets can be created with `UnixStream::pair`.
///
/// To shut down the stream in the write direction, you can call the
@@ -32,6 +32,7 @@
/// the stream in one direction.
///
/// [`shutdown()`]: fn@crate::io::AsyncWriteExt::shutdown
+ /// [`UnixListener::accept`]: crate::net::UnixListener::accept
pub struct UnixStream {
io: PollEvented<mio::net::UnixStream>,
}
@@ -65,6 +66,12 @@
/// can be used to concurrently read / write to the same socket on a single
/// task without splitting the socket.
///
+ /// The function may complete without the socket being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Cancel safety
///
/// This method is cancel safe. Once a readiness event occurs, the method
@@ -239,8 +246,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
- /// and will no longer yield data. If the stream is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The stream's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the stream is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
///
/// # Examples
@@ -656,7 +667,7 @@
/// Tries to read or write from the socket using a user-provided IO operation.
///
/// If the socket is ready, the provided closure is called. The closure
- /// should attempt to perform IO operation from the socket by manually
+ /// should attempt to perform IO operation on the socket by manually
/// calling the appropriate syscall. If the operation fails because the
/// socket is not actually ready, then the closure should return a
/// `WouldBlock` error and the readiness flag is cleared. The return value
@@ -675,6 +686,11 @@
/// defined on the Tokio `UnixStream` type, as this will mess with the
/// readiness flag and can cause the socket to behave incorrectly.
///
+ /// This method is not intended to be used with combined interests.
+ /// The closure should perform only one type of IO operation, so it should not
+ /// require more than one ready state. This method may panic or sleep forever
+ /// if it is called with a combined interest.
+ ///
/// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
///
/// [`readable()`]: UnixStream::readable()
@@ -685,23 +701,49 @@
interest: Interest,
f: impl FnOnce() -> io::Result<R>,
) -> io::Result<R> {
- self.io.registration().try_io(interest, f)
+ self.io
+ .registration()
+ .try_io(interest, || self.io.try_io(f))
}
/// Creates new `UnixStream` from a `std::os::unix::net::UnixStream`.
///
/// This function is intended to be used to wrap a UnixStream from the
- /// standard library in the Tokio equivalent. The conversion assumes
- /// nothing about the underlying stream; it is left up to the user to set
- /// it in non-blocking mode.
+ /// standard library in the Tokio equivalent.
+ ///
+ /// # Notes
+ ///
+ /// The caller is responsible for ensuring that the stream is in
+ /// non-blocking mode. Otherwise all I/O operations on the stream
+ /// will block the thread, which will cause unexpected behavior.
+ /// Non-blocking mode can be set using [`set_nonblocking`].
+ ///
+ /// [`set_nonblocking`]: std::os::unix::net::UnixStream::set_nonblocking
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::UnixStream;
+ /// use std::os::unix::net::UnixStream as StdUnixStream;
+ /// # use std::error::Error;
+ ///
+ /// # async fn dox() -> Result<(), Box<dyn Error>> {
+ /// let std_stream = StdUnixStream::connect("/path/to/the/socket")?;
+ /// std_stream.set_nonblocking(true)?;
+ /// let stream = UnixStream::from_std(std_stream)?;
+ /// # Ok(())
+ /// # }
+ /// ```
///
/// # Panics
///
- /// This function panics if thread-local runtime is not set.
+ /// This function panics if it is not called from within a runtime with
+ /// IO enabled.
///
/// The runtime is usually set implicitly when this function is called
/// from a future driven by a tokio runtime, otherwise runtime can be set
/// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+ #[track_caller]
pub fn from_std(stream: net::UnixStream) -> io::Result<UnixStream> {
let stream = mio::net::UnixStream::from_std(stream);
let io = PollEvented::new(stream)?;
diff --git a/src/net/unix/ucred.rs b/src/net/unix/ucred.rs
index 865303b..3cb61d7 100644
--- a/src/net/unix/ucred.rs
+++ b/src/net/unix/ucred.rs
@@ -1,24 +1,24 @@
-use libc::{gid_t, pid_t, uid_t};
+use crate::net::unix;
/// Credentials of a process.
#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
pub struct UCred {
/// PID (process ID) of the process.
- pid: Option<pid_t>,
+ pid: Option<unix::pid_t>,
/// UID (user ID) of the process.
- uid: uid_t,
+ uid: unix::uid_t,
/// GID (group ID) of the process.
- gid: gid_t,
+ gid: unix::gid_t,
}
impl UCred {
/// Gets UID (user ID) of the process.
- pub fn uid(&self) -> uid_t {
+ pub fn uid(&self) -> unix::uid_t {
self.uid
}
/// Gets GID (group ID) of the process.
- pub fn gid(&self) -> gid_t {
+ pub fn gid(&self) -> unix::gid_t {
self.gid
}
@@ -26,7 +26,7 @@
///
/// This is only implemented under Linux, Android, iOS, macOS, Solaris and
/// Illumos. On other platforms this will always return `None`.
- pub fn pid(&self) -> Option<pid_t> {
+ pub fn pid(&self) -> Option<unix::pid_t> {
self.pid
}
}
@@ -48,7 +48,7 @@
#[cfg(any(target_os = "linux", target_os = "android", target_os = "openbsd"))]
pub(crate) mod impl_linux {
- use crate::net::unix::UnixStream;
+ use crate::net::unix::{self, UnixStream};
use libc::{c_void, getsockopt, socklen_t, SOL_SOCKET, SO_PEERCRED};
use std::{io, mem};
@@ -87,9 +87,9 @@
);
if ret == 0 && ucred_size as usize == mem::size_of::<ucred>() {
Ok(super::UCred {
- uid: ucred.uid,
- gid: ucred.gid,
- pid: Some(ucred.pid),
+ uid: ucred.uid as unix::uid_t,
+ gid: ucred.gid as unix::gid_t,
+ pid: Some(ucred.pid as unix::pid_t),
})
} else {
Err(io::Error::last_os_error())
@@ -100,7 +100,7 @@
#[cfg(any(target_os = "netbsd"))]
pub(crate) mod impl_netbsd {
- use crate::net::unix::UnixStream;
+ use crate::net::unix::{self, UnixStream};
use libc::{c_void, getsockopt, socklen_t, unpcbid, LOCAL_PEEREID, SOL_SOCKET};
use std::io;
@@ -129,9 +129,9 @@
);
if ret == 0 && unpcbid_size as usize == size_of::<unpcbid>() {
Ok(super::UCred {
- uid: unpcbid.unp_euid,
- gid: unpcbid.unp_egid,
- pid: Some(unpcbid.unp_pid),
+ uid: unpcbid.unp_euid as unix::uid_t,
+ gid: unpcbid.unp_egid as unix::gid_t,
+ pid: Some(unpcbid.unp_pid as unix::pid_t),
})
} else {
Err(io::Error::last_os_error())
@@ -142,7 +142,7 @@
#[cfg(any(target_os = "dragonfly", target_os = "freebsd"))]
pub(crate) mod impl_bsd {
- use crate::net::unix::UnixStream;
+ use crate::net::unix::{self, UnixStream};
use libc::getpeereid;
use std::io;
@@ -160,8 +160,8 @@
if ret == 0 {
Ok(super::UCred {
- uid: uid.assume_init(),
- gid: gid.assume_init(),
+ uid: uid.assume_init() as unix::uid_t,
+ gid: gid.assume_init() as unix::gid_t,
pid: None,
})
} else {
@@ -173,7 +173,7 @@
#[cfg(any(target_os = "macos", target_os = "ios"))]
pub(crate) mod impl_macos {
- use crate::net::unix::UnixStream;
+ use crate::net::unix::{self, UnixStream};
use libc::{c_void, getpeereid, getsockopt, pid_t, LOCAL_PEEREPID, SOL_LOCAL};
use std::io;
@@ -207,9 +207,9 @@
if ret == 0 {
Ok(super::UCred {
- uid: uid.assume_init(),
- gid: gid.assume_init(),
- pid: Some(pid.assume_init()),
+ uid: uid.assume_init() as unix::uid_t,
+ gid: gid.assume_init() as unix::gid_t,
+ pid: Some(pid.assume_init() as unix::pid_t),
})
} else {
Err(io::Error::last_os_error())
@@ -220,7 +220,7 @@
#[cfg(any(target_os = "solaris", target_os = "illumos"))]
pub(crate) mod impl_solaris {
- use crate::net::unix::UnixStream;
+ use crate::net::unix::{self, UnixStream};
use std::io;
use std::os::unix::io::AsRawFd;
use std::ptr;
@@ -240,9 +240,9 @@
libc::ucred_free(cred);
Ok(super::UCred {
- uid,
- gid,
- pid: Some(pid),
+ uid: uid as unix::uid_t,
+ gid: gid as unix::gid_t,
+ pid: Some(pid as unix::pid_t),
})
} else {
Err(io::Error::last_os_error())
diff --git a/src/net/windows/named_pipe.rs b/src/net/windows/named_pipe.rs
index 550fd4d..9ede94e 100644
--- a/src/net/windows/named_pipe.rs
+++ b/src/net/windows/named_pipe.rs
@@ -12,25 +12,26 @@
use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready};
use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle};
+cfg_io_util! {
+ use bytes::BufMut;
+}
+
// Hide imports which are not used when generating documentation.
#[cfg(not(docsrs))]
mod doc {
pub(super) use crate::os::windows::ffi::OsStrExt;
- pub(super) use crate::winapi::shared::minwindef::{DWORD, FALSE};
- pub(super) use crate::winapi::um::fileapi;
- pub(super) use crate::winapi::um::handleapi;
- pub(super) use crate::winapi::um::namedpipeapi;
- pub(super) use crate::winapi::um::winbase;
- pub(super) use crate::winapi::um::winnt;
-
+ pub(super) mod windows_sys {
+ pub(crate) use windows_sys::{
+ Win32::Foundation::*, Win32::Storage::FileSystem::*, Win32::System::Pipes::*,
+ Win32::System::SystemServices::*,
+ };
+ }
pub(super) use mio::windows as mio_windows;
}
// NB: none of these shows up in public API, so don't document them.
#[cfg(docsrs)]
mod doc {
- pub type DWORD = crate::doc::NotDefinedHere;
-
pub(super) mod mio_windows {
pub type NamedPipe = crate::doc::NotDefinedHere;
}
@@ -97,7 +98,6 @@
/// # Ok(()) }
/// ```
///
-/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY
/// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
#[derive(Debug)]
pub struct NamedPipeServer {
@@ -188,17 +188,15 @@
/// # Ok(()) }
/// ```
pub async fn connect(&self) -> io::Result<()> {
- loop {
- match self.io.connect() {
- Ok(()) => break,
- Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
- self.io.registration().readiness(Interest::WRITABLE).await?;
- }
- Err(e) => return Err(e),
+ match self.io.connect() {
+ Err(e) if e.kind() == io::ErrorKind::WouldBlock => {
+ self.io
+ .registration()
+ .async_io(Interest::WRITABLE, || self.io.connect())
+ .await
}
+ x => x,
}
-
- Ok(())
}
/// Disconnects the server end of a named pipe instance from a client
@@ -207,7 +205,7 @@
/// ```
/// use tokio::io::AsyncWriteExt;
/// use tokio::net::windows::named_pipe::{ClientOptions, ServerOptions};
- /// use winapi::shared::winerror;
+ /// use windows_sys::Win32::Foundation::ERROR_PIPE_NOT_CONNECTED;
///
/// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-disconnect";
///
@@ -227,7 +225,7 @@
/// // Write fails with an OS-specific error after client has been
/// // disconnected.
/// let e = client.write(b"ping").await.unwrap_err();
- /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_NOT_CONNECTED as i32));
+ /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_NOT_CONNECTED as i32));
/// # Ok(()) }
/// ```
pub fn disconnect(&self) -> io::Result<()> {
@@ -240,6 +238,12 @@
/// can be used to concurrently read / write to the same pipe on a single
/// task without splitting the pipe.
///
+ /// The function may complete without the pipe being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Examples
///
/// Concurrently read and write to the pipe on the same task without
@@ -399,8 +403,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the pipe's read half is closed
- /// and will no longer yield data. If the pipe is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The pipe's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the pipe is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
///
/// # Examples
@@ -528,6 +536,86 @@
.try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs))
}
+ cfg_io_util! {
+ /// Tries to read data from the stream into the provided buffer, advancing the
+ /// buffer's internal cursor, returning how many bytes were read.
+ ///
+ /// Receives any pending data from the pipe but does not wait for new data
+ /// to arrive. On success, returns the number of bytes read. Because
+ /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by
+ /// the async task and can exist entirely on the stack.
+ ///
+ /// Usually, [`readable()`] or [`ready()`] is used with this function.
+ ///
+ /// [`readable()`]: NamedPipeServer::readable()
+ /// [`ready()`]: NamedPipeServer::ready()
+ ///
+ /// # Return
+ ///
+ /// If data is successfully read, `Ok(n)` is returned, where `n` is the
+ /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
+ /// and will no longer yield data. If the stream is not ready to read data
+ /// `Err(io::ErrorKind::WouldBlock)` is returned.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::windows::named_pipe;
+ /// use std::error::Error;
+ /// use std::io;
+ ///
+ /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-client-readable";
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn Error>> {
+ /// let server = named_pipe::ServerOptions::new().create(PIPE_NAME)?;
+ ///
+ /// loop {
+ /// // Wait for the pipe to be readable
+ /// server.readable().await?;
+ ///
+ /// let mut buf = Vec::with_capacity(4096);
+ ///
+ /// // Try to read data, this may still fail with `WouldBlock`
+ /// // if the readiness event is a false positive.
+ /// match server.try_read_buf(&mut buf) {
+ /// Ok(0) => break,
+ /// Ok(n) => {
+ /// println!("read {} bytes", n);
+ /// }
+ /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+ /// continue;
+ /// }
+ /// Err(e) => {
+ /// return Err(e.into());
+ /// }
+ /// }
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn try_read_buf<B: BufMut>(&self, buf: &mut B) -> io::Result<usize> {
+ self.io.registration().try_io(Interest::READABLE, || {
+ use std::io::Read;
+
+ let dst = buf.chunk_mut();
+ let dst =
+ unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]) };
+
+ // Safety: We trust `NamedPipeServer::read` to have filled up `n` bytes in the
+ // buffer.
+ let n = (&*self.io).read(dst)?;
+
+ unsafe {
+ buf.advance_mut(n);
+ }
+
+ Ok(n)
+ })
+ }
+ }
+
/// Waits for the pipe to become writable.
///
/// This function is equivalent to `ready(Interest::WRITABLE)` and is usually
@@ -724,27 +812,32 @@
.try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf))
}
- /// Tries to read or write from the socket using a user-provided IO operation.
+ /// Tries to read or write from the pipe using a user-provided IO operation.
///
- /// If the socket is ready, the provided closure is called. The closure
- /// should attempt to perform IO operation from the socket by manually
+ /// If the pipe is ready, the provided closure is called. The closure
+ /// should attempt to perform IO operation from the pipe by manually
/// calling the appropriate syscall. If the operation fails because the
- /// socket is not actually ready, then the closure should return a
+ /// pipe is not actually ready, then the closure should return a
/// `WouldBlock` error and the readiness flag is cleared. The return value
/// of the closure is then returned by `try_io`.
///
- /// If the socket is not ready, then the closure is not called
+ /// If the pipe is not ready, then the closure is not called
/// and a `WouldBlock` error is returned.
///
/// The closure should only return a `WouldBlock` error if it has performed
- /// an IO operation on the socket that failed due to the socket not being
+ /// an IO operation on the pipe that failed due to the pipe not being
/// ready. Returning a `WouldBlock` error in any other situation will
- /// incorrectly clear the readiness flag, which can cause the socket to
+ /// incorrectly clear the readiness flag, which can cause the pipe to
/// behave incorrectly.
///
/// The closure should not perform the IO operation using any of the
/// methods defined on the Tokio `NamedPipeServer` type, as this will mess with
- /// the readiness flag and can cause the socket to behave incorrectly.
+ /// the readiness flag and can cause the pipe to behave incorrectly.
+ ///
+ /// This method is not intended to be used with combined interests.
+ /// The closure should perform only one type of IO operation, so it should not
+ /// require more than one ready state. This method may panic or sleep forever
+ /// if it is called with a combined interest.
///
/// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
///
@@ -819,7 +912,7 @@
/// use std::time::Duration;
/// use tokio::net::windows::named_pipe::ClientOptions;
/// use tokio::time;
-/// use winapi::shared::winerror;
+/// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
///
/// const PIPE_NAME: &str = r"\\.\pipe\named-pipe-idiomatic-client";
///
@@ -827,7 +920,7 @@
/// let client = loop {
/// match ClientOptions::new().open(PIPE_NAME) {
/// Ok(client) => break client,
-/// Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+/// Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
/// Err(e) => return Err(e),
/// }
///
@@ -838,7 +931,7 @@
/// # Ok(()) }
/// ```
///
-/// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY
+/// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html
/// [Windows named pipe]: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
#[derive(Debug)]
pub struct NamedPipeClient {
@@ -902,6 +995,12 @@
/// can be used to concurrently read / write to the same pipe on a single
/// task without splitting the pipe.
///
+ /// The function may complete without the pipe being ready. This is a
+ /// false-positive and attempting an operation will return with
+ /// `io::ErrorKind::WouldBlock`. The function can also return with an empty
+ /// [`Ready`] set, so you should always check the returned value and possibly
+ /// wait again if the requested states are not set.
+ ///
/// # Examples
///
/// Concurrently read and write to the pipe on the same task without
@@ -1059,8 +1158,12 @@
/// # Return
///
/// If data is successfully read, `Ok(n)` is returned, where `n` is the
- /// number of bytes read. `Ok(0)` indicates the pipe's read half is closed
- /// and will no longer yield data. If the pipe is not ready to read data
+ /// number of bytes read. If `n` is `0`, then it can indicate one of two scenarios:
+ ///
+ /// 1. The pipe's read half is closed and will no longer yield data.
+ /// 2. The specified buffer was 0 bytes in length.
+ ///
+ /// If the pipe is not ready to read data,
/// `Err(io::ErrorKind::WouldBlock)` is returned.
///
/// # Examples
@@ -1186,6 +1289,86 @@
.try_io(Interest::READABLE, || (&*self.io).read_vectored(bufs))
}
+ cfg_io_util! {
+ /// Tries to read data from the stream into the provided buffer, advancing the
+ /// buffer's internal cursor, returning how many bytes were read.
+ ///
+ /// Receives any pending data from the pipe but does not wait for new data
+ /// to arrive. On success, returns the number of bytes read. Because
+ /// `try_read_buf()` is non-blocking, the buffer does not have to be stored by
+ /// the async task and can exist entirely on the stack.
+ ///
+ /// Usually, [`readable()`] or [`ready()`] is used with this function.
+ ///
+ /// [`readable()`]: NamedPipeClient::readable()
+ /// [`ready()`]: NamedPipeClient::ready()
+ ///
+ /// # Return
+ ///
+ /// If data is successfully read, `Ok(n)` is returned, where `n` is the
+ /// number of bytes read. `Ok(0)` indicates the stream's read half is closed
+ /// and will no longer yield data. If the stream is not ready to read data
+ /// `Err(io::ErrorKind::WouldBlock)` is returned.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::net::windows::named_pipe;
+ /// use std::error::Error;
+ /// use std::io;
+ ///
+ /// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-client-readable";
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn Error>> {
+ /// let client = named_pipe::ClientOptions::new().open(PIPE_NAME)?;
+ ///
+ /// loop {
+ /// // Wait for the pipe to be readable
+ /// client.readable().await?;
+ ///
+ /// let mut buf = Vec::with_capacity(4096);
+ ///
+ /// // Try to read data, this may still fail with `WouldBlock`
+ /// // if the readiness event is a false positive.
+ /// match client.try_read_buf(&mut buf) {
+ /// Ok(0) => break,
+ /// Ok(n) => {
+ /// println!("read {} bytes", n);
+ /// }
+ /// Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
+ /// continue;
+ /// }
+ /// Err(e) => {
+ /// return Err(e.into());
+ /// }
+ /// }
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn try_read_buf<B: BufMut>(&self, buf: &mut B) -> io::Result<usize> {
+ self.io.registration().try_io(Interest::READABLE, || {
+ use std::io::Read;
+
+ let dst = buf.chunk_mut();
+ let dst =
+ unsafe { &mut *(dst as *mut _ as *mut [std::mem::MaybeUninit<u8>] as *mut [u8]) };
+
+ // Safety: We trust `NamedPipeClient::read` to have filled up `n` bytes in the
+ // buffer.
+ let n = (&*self.io).read(dst)?;
+
+ unsafe {
+ buf.advance_mut(n);
+ }
+
+ Ok(n)
+ })
+ }
+ }
+
/// Waits for the pipe to become writable.
///
/// This function is equivalent to `ready(Interest::WRITABLE)` and is usually
@@ -1379,27 +1562,32 @@
.try_io(Interest::WRITABLE, || (&*self.io).write_vectored(buf))
}
- /// Tries to read or write from the socket using a user-provided IO operation.
+ /// Tries to read or write from the pipe using a user-provided IO operation.
///
- /// If the socket is ready, the provided closure is called. The closure
- /// should attempt to perform IO operation from the socket by manually
+ /// If the pipe is ready, the provided closure is called. The closure
+ /// should attempt to perform IO operation from the pipe by manually
/// calling the appropriate syscall. If the operation fails because the
- /// socket is not actually ready, then the closure should return a
+ /// pipe is not actually ready, then the closure should return a
/// `WouldBlock` error and the readiness flag is cleared. The return value
/// of the closure is then returned by `try_io`.
///
- /// If the socket is not ready, then the closure is not called
+ /// If the pipe is not ready, then the closure is not called
/// and a `WouldBlock` error is returned.
///
/// The closure should only return a `WouldBlock` error if it has performed
- /// an IO operation on the socket that failed due to the socket not being
+ /// an IO operation on the pipe that failed due to the pipe not being
/// ready. Returning a `WouldBlock` error in any other situation will
- /// incorrectly clear the readiness flag, which can cause the socket to
+ /// incorrectly clear the readiness flag, which can cause the pipe to
/// behave incorrectly.
///
/// The closure should not perform the IO operation using any of the methods
/// defined on the Tokio `NamedPipeClient` type, as this will mess with the
- /// readiness flag and can cause the socket to behave incorrectly.
+ /// readiness flag and can cause the pipe to behave incorrectly.
+ ///
+ /// This method is not intended to be used with combined interests.
+ /// The closure should perform only one type of IO operation, so it should not
+ /// require more than one ready state. This method may panic or sleep forever
+ /// if it is called with a combined interest.
///
/// Usually, [`readable()`], [`writable()`] or [`ready()`] is used with this function.
///
@@ -1477,12 +1665,12 @@
/// See [`ServerOptions::create`].
#[derive(Debug, Clone)]
pub struct ServerOptions {
- open_mode: DWORD,
- pipe_mode: DWORD,
- max_instances: DWORD,
- out_buffer_size: DWORD,
- in_buffer_size: DWORD,
- default_timeout: DWORD,
+ open_mode: u32,
+ pipe_mode: u32,
+ max_instances: u32,
+ out_buffer_size: u32,
+ in_buffer_size: u32,
+ default_timeout: u32,
}
impl ServerOptions {
@@ -1499,9 +1687,9 @@
/// ```
pub fn new() -> ServerOptions {
ServerOptions {
- open_mode: winbase::PIPE_ACCESS_DUPLEX | winbase::FILE_FLAG_OVERLAPPED,
- pipe_mode: winbase::PIPE_TYPE_BYTE | winbase::PIPE_REJECT_REMOTE_CLIENTS,
- max_instances: winbase::PIPE_UNLIMITED_INSTANCES,
+ open_mode: windows_sys::PIPE_ACCESS_DUPLEX | windows_sys::FILE_FLAG_OVERLAPPED,
+ pipe_mode: windows_sys::PIPE_TYPE_BYTE | windows_sys::PIPE_REJECT_REMOTE_CLIENTS,
+ max_instances: windows_sys::PIPE_UNLIMITED_INSTANCES,
out_buffer_size: 65536,
in_buffer_size: 65536,
default_timeout: 0,
@@ -1517,11 +1705,10 @@
///
/// [`dwPipeMode`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
pub fn pipe_mode(&mut self, pipe_mode: PipeMode) -> &mut Self {
- self.pipe_mode = match pipe_mode {
- PipeMode::Byte => winbase::PIPE_TYPE_BYTE,
- PipeMode::Message => winbase::PIPE_TYPE_MESSAGE,
- };
-
+ let is_msg = matches!(pipe_mode, PipeMode::Message);
+ // Pipe mode is implemented as a bit flag 0x4. Set is message and unset
+ // is byte.
+ bool_flag!(self.pipe_mode, is_msg, windows_sys::PIPE_TYPE_MESSAGE);
self
}
@@ -1617,7 +1804,7 @@
/// # Ok(()) }
/// ```
pub fn access_inbound(&mut self, allowed: bool) -> &mut Self {
- bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_INBOUND);
+ bool_flag!(self.open_mode, allowed, windows_sys::PIPE_ACCESS_INBOUND);
self
}
@@ -1715,7 +1902,7 @@
/// # Ok(()) }
/// ```
pub fn access_outbound(&mut self, allowed: bool) -> &mut Self {
- bool_flag!(self.open_mode, allowed, winbase::PIPE_ACCESS_OUTBOUND);
+ bool_flag!(self.open_mode, allowed, windows_sys::PIPE_ACCESS_OUTBOUND);
self
}
@@ -1786,7 +1973,113 @@
bool_flag!(
self.open_mode,
first,
- winbase::FILE_FLAG_FIRST_PIPE_INSTANCE
+ windows_sys::FILE_FLAG_FIRST_PIPE_INSTANCE
+ );
+ self
+ }
+
+ /// Requests permission to modify the pipe's discretionary access control list.
+ ///
+ /// This corresponds to setting [`WRITE_DAC`] in dwOpenMode.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use std::{io, os::windows::prelude::AsRawHandle, ptr};
+ //
+ /// use tokio::net::windows::named_pipe::ServerOptions;
+ /// use windows_sys::{
+ /// Win32::Foundation::ERROR_SUCCESS,
+ /// Win32::Security::DACL_SECURITY_INFORMATION,
+ /// Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT},
+ /// };
+ ///
+ /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe";
+ ///
+ /// # #[tokio::main] async fn main() -> io::Result<()> {
+ /// let mut pipe_template = ServerOptions::new();
+ /// pipe_template.write_dac(true);
+ /// let pipe = pipe_template.create(PIPE_NAME)?;
+ ///
+ /// unsafe {
+ /// assert_eq!(
+ /// ERROR_SUCCESS,
+ /// SetSecurityInfo(
+ /// pipe.as_raw_handle() as _,
+ /// SE_KERNEL_OBJECT,
+ /// DACL_SECURITY_INFORMATION,
+ /// ptr::null_mut(),
+ /// ptr::null_mut(),
+ /// ptr::null_mut(),
+ /// ptr::null_mut(),
+ /// )
+ /// );
+ /// }
+ ///
+ /// # Ok(()) }
+ /// ```
+ ///
+ /// ```
+ /// use std::{io, os::windows::prelude::AsRawHandle, ptr};
+ //
+ /// use tokio::net::windows::named_pipe::ServerOptions;
+ /// use windows_sys::{
+ /// Win32::Foundation::ERROR_ACCESS_DENIED,
+ /// Win32::Security::DACL_SECURITY_INFORMATION,
+ /// Win32::Security::Authorization::{SetSecurityInfo, SE_KERNEL_OBJECT},
+ /// };
+ ///
+ /// const PIPE_NAME: &str = r"\\.\pipe\write_dac_pipe_fail";
+ ///
+ /// # #[tokio::main] async fn main() -> io::Result<()> {
+ /// let mut pipe_template = ServerOptions::new();
+ /// pipe_template.write_dac(false);
+ /// let pipe = pipe_template.create(PIPE_NAME)?;
+ ///
+ /// unsafe {
+ /// assert_eq!(
+ /// ERROR_ACCESS_DENIED,
+ /// SetSecurityInfo(
+ /// pipe.as_raw_handle() as _,
+ /// SE_KERNEL_OBJECT,
+ /// DACL_SECURITY_INFORMATION,
+ /// ptr::null_mut(),
+ /// ptr::null_mut(),
+ /// ptr::null_mut(),
+ /// ptr::null_mut(),
+ /// )
+ /// );
+ /// }
+ ///
+ /// # Ok(()) }
+ /// ```
+ ///
+ /// [`WRITE_DAC`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
+ pub fn write_dac(&mut self, requested: bool) -> &mut Self {
+ bool_flag!(self.open_mode, requested, windows_sys::WRITE_DAC);
+ self
+ }
+
+ /// Requests permission to modify the pipe's owner.
+ ///
+ /// This corresponds to setting [`WRITE_OWNER`] in dwOpenMode.
+ ///
+ /// [`WRITE_OWNER`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
+ pub fn write_owner(&mut self, requested: bool) -> &mut Self {
+ bool_flag!(self.open_mode, requested, windows_sys::WRITE_OWNER);
+ self
+ }
+
+ /// Requests permission to modify the pipe's system access control list.
+ ///
+ /// This corresponds to setting [`ACCESS_SYSTEM_SECURITY`] in dwOpenMode.
+ ///
+ /// [`ACCESS_SYSTEM_SECURITY`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
+ pub fn access_system_security(&mut self, requested: bool) -> &mut Self {
+ bool_flag!(
+ self.open_mode,
+ requested,
+ windows_sys::ACCESS_SYSTEM_SECURITY
);
self
}
@@ -1798,7 +2091,11 @@
///
/// [`PIPE_REJECT_REMOTE_CLIENTS`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea#pipe_reject_remote_clients
pub fn reject_remote_clients(&mut self, reject: bool) -> &mut Self {
- bool_flag!(self.pipe_mode, reject, winbase::PIPE_REJECT_REMOTE_CLIENTS);
+ bool_flag!(
+ self.pipe_mode,
+ reject,
+ windows_sys::PIPE_REJECT_REMOTE_CLIENTS
+ );
self
}
@@ -1820,7 +2117,7 @@
/// ```
/// use std::io;
/// use tokio::net::windows::named_pipe::{ServerOptions, ClientOptions};
- /// use winapi::shared::winerror;
+ /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
///
/// const PIPE_NAME: &str = r"\\.\pipe\tokio-named-pipe-max-instances";
///
@@ -1836,11 +2133,11 @@
///
/// // Too many servers!
/// let e = server.create(PIPE_NAME).unwrap_err();
- /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32));
+ /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32));
///
/// // Still too many servers even if we specify a higher value!
/// let e = server.max_instances(100).create(PIPE_NAME).unwrap_err();
- /// assert_eq!(e.raw_os_error(), Some(winerror::ERROR_PIPE_BUSY as i32));
+ /// assert_eq!(e.raw_os_error(), Some(ERROR_PIPE_BUSY as i32));
/// # Ok(()) }
/// ```
///
@@ -1856,9 +2153,10 @@
/// let builder = ServerOptions::new().max_instances(255);
/// # Ok(()) }
/// ```
+ #[track_caller]
pub fn max_instances(&mut self, instances: usize) -> &mut Self {
assert!(instances < 255, "cannot specify more than 254 instances");
- self.max_instances = instances as DWORD;
+ self.max_instances = instances as u32;
self
}
@@ -1868,7 +2166,7 @@
///
/// [`nOutBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
pub fn out_buffer_size(&mut self, buffer: u32) -> &mut Self {
- self.out_buffer_size = buffer as DWORD;
+ self.out_buffer_size = buffer;
self
}
@@ -1878,7 +2176,7 @@
///
/// [`nInBufferSize`]: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea
pub fn in_buffer_size(&mut self, buffer: u32) -> &mut Self {
- self.in_buffer_size = buffer as DWORD;
+ self.in_buffer_size = buffer;
self
}
@@ -1935,7 +2233,7 @@
///
/// [`create`]: ServerOptions::create
/// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
- /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES
+ /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html
pub unsafe fn create_with_security_attributes_raw(
&self,
addr: impl AsRef<OsStr>,
@@ -1943,7 +2241,7 @@
) -> io::Result<NamedPipeServer> {
let addr = encode_addr(addr);
- let h = namedpipeapi::CreateNamedPipeW(
+ let h = windows_sys::CreateNamedPipeW(
addr.as_ptr(),
self.open_mode,
self.pipe_mode,
@@ -1954,11 +2252,11 @@
attrs as *mut _,
);
- if h == handleapi::INVALID_HANDLE_VALUE {
+ if h == windows_sys::INVALID_HANDLE_VALUE {
return Err(io::Error::last_os_error());
}
- NamedPipeServer::from_raw_handle(h)
+ NamedPipeServer::from_raw_handle(h as _)
}
}
@@ -1968,8 +2266,8 @@
/// See [`ClientOptions::open`].
#[derive(Debug, Clone)]
pub struct ClientOptions {
- desired_access: DWORD,
- security_qos_flags: DWORD,
+ desired_access: u32,
+ security_qos_flags: u32,
}
impl ClientOptions {
@@ -1988,8 +2286,9 @@
/// ```
pub fn new() -> Self {
Self {
- desired_access: winnt::GENERIC_READ | winnt::GENERIC_WRITE,
- security_qos_flags: winbase::SECURITY_IDENTIFICATION | winbase::SECURITY_SQOS_PRESENT,
+ desired_access: windows_sys::GENERIC_READ | windows_sys::GENERIC_WRITE,
+ security_qos_flags: windows_sys::SECURITY_IDENTIFICATION
+ | windows_sys::SECURITY_SQOS_PRESENT,
}
}
@@ -2000,7 +2299,7 @@
/// [`GENERIC_READ`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights
/// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
pub fn read(&mut self, allowed: bool) -> &mut Self {
- bool_flag!(self.desired_access, allowed, winnt::GENERIC_READ);
+ bool_flag!(self.desired_access, allowed, windows_sys::GENERIC_READ);
self
}
@@ -2011,7 +2310,7 @@
/// [`GENERIC_WRITE`]: https://docs.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights
/// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
pub fn write(&mut self, allowed: bool) -> &mut Self {
- bool_flag!(self.desired_access, allowed, winnt::GENERIC_WRITE);
+ bool_flag!(self.desired_access, allowed, windows_sys::GENERIC_WRITE);
self
}
@@ -2034,11 +2333,11 @@
/// automatically when using this method.
///
/// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea
- /// [`SECURITY_IDENTIFICATION`]: crate::winapi::um::winbase::SECURITY_IDENTIFICATION
+ /// [`SECURITY_IDENTIFICATION`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Storage/FileSystem/constant.SECURITY_IDENTIFICATION.html
/// [Impersonation Levels]: https://docs.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level
pub fn security_qos_flags(&mut self, flags: u32) -> &mut Self {
// See: https://github.com/rust-lang/rust/pull/58216
- self.security_qos_flags = flags | winbase::SECURITY_SQOS_PRESENT;
+ self.security_qos_flags = flags | windows_sys::SECURITY_SQOS_PRESENT;
self
}
@@ -2063,8 +2362,7 @@
/// but the server is not currently waiting for a connection. Please see the
/// examples for how to check for this error.
///
- /// [`ERROR_PIPE_BUSY`]: crate::winapi::shared::winerror::ERROR_PIPE_BUSY
- /// [`winapi`]: crate::winapi
+ /// [`ERROR_PIPE_BUSY`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_PIPE_BUSY.html
/// [enabled I/O]: crate::runtime::Builder::enable_io
/// [Tokio Runtime]: crate::runtime::Runtime
///
@@ -2075,7 +2373,7 @@
/// use std::time::Duration;
/// use tokio::net::windows::named_pipe::ClientOptions;
/// use tokio::time;
- /// use winapi::shared::winerror;
+ /// use windows_sys::Win32::Foundation::ERROR_PIPE_BUSY;
///
/// const PIPE_NAME: &str = r"\\.\pipe\mynamedpipe";
///
@@ -2083,7 +2381,7 @@
/// let client = loop {
/// match ClientOptions::new().open(PIPE_NAME) {
/// Ok(client) => break client,
- /// Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+ /// Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
/// Err(e) => return Err(e),
/// }
///
@@ -2113,7 +2411,7 @@
///
/// [`open`]: ClientOptions::open
/// [`CreateFile`]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
- /// [`SECURITY_ATTRIBUTES`]: crate::winapi::um::minwinbase::SECURITY_ATTRIBUTES
+ /// [`SECURITY_ATTRIBUTES`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Security/struct.SECURITY_ATTRIBUTES.html
pub unsafe fn open_with_security_attributes_raw(
&self,
addr: impl AsRef<OsStr>,
@@ -2122,28 +2420,28 @@
let addr = encode_addr(addr);
// NB: We could use a platform specialized `OpenOptions` here, but since
- // we have access to winapi it ultimately doesn't hurt to use
+ // we have access to windows_sys it ultimately doesn't hurt to use
// `CreateFile` explicitly since it allows the use of our already
// well-structured wide `addr` to pass into CreateFileW.
- let h = fileapi::CreateFileW(
+ let h = windows_sys::CreateFileW(
addr.as_ptr(),
self.desired_access,
0,
attrs as *mut _,
- fileapi::OPEN_EXISTING,
+ windows_sys::OPEN_EXISTING,
self.get_flags(),
- ptr::null_mut(),
+ 0,
);
- if h == handleapi::INVALID_HANDLE_VALUE {
+ if h == windows_sys::INVALID_HANDLE_VALUE {
return Err(io::Error::last_os_error());
}
- NamedPipeClient::from_raw_handle(h)
+ NamedPipeClient::from_raw_handle(h as _)
}
fn get_flags(&self) -> u32 {
- self.security_qos_flags | winbase::FILE_FLAG_OVERLAPPED
+ self.security_qos_flags | windows_sys::FILE_FLAG_OVERLAPPED
}
}
@@ -2156,16 +2454,19 @@
/// Data is written to the pipe as a stream of bytes. The pipe does not
/// distinguish bytes written during different write operations.
///
- /// Corresponds to [`PIPE_TYPE_BYTE`][crate::winapi::um::winbase::PIPE_TYPE_BYTE].
+ /// Corresponds to [`PIPE_TYPE_BYTE`].
+ ///
+ /// [`PIPE_TYPE_BYTE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_BYTE.html
Byte,
/// Data is written to the pipe as a stream of messages. The pipe treats the
/// bytes written during each write operation as a message unit. Any reading
/// on a named pipe returns [`ERROR_MORE_DATA`] when a message is not read
/// completely.
///
- /// Corresponds to [`PIPE_TYPE_MESSAGE`][crate::winapi::um::winbase::PIPE_TYPE_MESSAGE].
+ /// Corresponds to [`PIPE_TYPE_MESSAGE`].
///
- /// [`ERROR_MORE_DATA`]: crate::winapi::shared::winerror::ERROR_MORE_DATA
+ /// [`ERROR_MORE_DATA`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/Foundation/constant.ERROR_MORE_DATA.html
+ /// [`PIPE_TYPE_MESSAGE`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_TYPE_MESSAGE.html
Message,
}
@@ -2175,11 +2476,15 @@
pub enum PipeEnd {
/// The named pipe refers to the client end of a named pipe instance.
///
- /// Corresponds to [`PIPE_CLIENT_END`][crate::winapi::um::winbase::PIPE_CLIENT_END].
+ /// Corresponds to [`PIPE_CLIENT_END`].
+ ///
+ /// [`PIPE_CLIENT_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_CLIENT_END.html
Client,
/// The named pipe refers to the server end of a named pipe instance.
///
- /// Corresponds to [`PIPE_SERVER_END`][crate::winapi::um::winbase::PIPE_SERVER_END].
+ /// Corresponds to [`PIPE_SERVER_END`].
+ ///
+ /// [`PIPE_SERVER_END`]: https://docs.rs/windows-sys/latest/windows_sys/Win32/System/Pipes/constant.PIPE_SERVER_END.html
Server,
}
@@ -2217,26 +2522,26 @@
let mut in_buffer_size = 0;
let mut max_instances = 0;
- let result = namedpipeapi::GetNamedPipeInfo(
- handle,
+ let result = windows_sys::GetNamedPipeInfo(
+ handle as _,
&mut flags,
&mut out_buffer_size,
&mut in_buffer_size,
&mut max_instances,
);
- if result == FALSE {
+ if result == 0 {
return Err(io::Error::last_os_error());
}
let mut end = PipeEnd::Client;
let mut mode = PipeMode::Byte;
- if flags & winbase::PIPE_SERVER_END != 0 {
+ if flags & windows_sys::PIPE_SERVER_END != 0 {
end = PipeEnd::Server;
}
- if flags & winbase::PIPE_TYPE_MESSAGE != 0 {
+ if flags & windows_sys::PIPE_TYPE_MESSAGE != 0 {
mode = PipeMode::Message;
}
@@ -2248,3 +2553,48 @@
max_instances,
})
}
+
+#[cfg(test)]
+mod test {
+ use self::windows_sys::{PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_TYPE_MESSAGE};
+ use super::*;
+
+ #[test]
+ fn opts_default_pipe_mode() {
+ let opts = ServerOptions::new();
+ assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS);
+ }
+
+ #[test]
+ fn opts_unset_reject_remote() {
+ let mut opts = ServerOptions::new();
+ opts.reject_remote_clients(false);
+ assert_eq!(opts.pipe_mode & PIPE_REJECT_REMOTE_CLIENTS, 0);
+ }
+
+ #[test]
+ fn opts_set_pipe_mode_maintains_reject_remote_clients() {
+ let mut opts = ServerOptions::new();
+ opts.pipe_mode(PipeMode::Byte);
+ assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS);
+
+ opts.reject_remote_clients(false);
+ opts.pipe_mode(PipeMode::Byte);
+ assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE);
+
+ opts.reject_remote_clients(true);
+ opts.pipe_mode(PipeMode::Byte);
+ assert_eq!(opts.pipe_mode, PIPE_TYPE_BYTE | PIPE_REJECT_REMOTE_CLIENTS);
+
+ opts.reject_remote_clients(false);
+ opts.pipe_mode(PipeMode::Message);
+ assert_eq!(opts.pipe_mode, PIPE_TYPE_MESSAGE);
+
+ opts.reject_remote_clients(true);
+ opts.pipe_mode(PipeMode::Message);
+ assert_eq!(
+ opts.pipe_mode,
+ PIPE_TYPE_MESSAGE | PIPE_REJECT_REMOTE_CLIENTS
+ );
+ }
+}
diff --git a/src/park/either.rs b/src/park/either.rs
deleted file mode 100644
index ee02ec1..0000000
--- a/src/park/either.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-#![cfg_attr(not(feature = "full"), allow(dead_code))]
-
-use crate::park::{Park, Unpark};
-
-use std::fmt;
-use std::time::Duration;
-
-pub(crate) enum Either<A, B> {
- A(A),
- B(B),
-}
-
-impl<A, B> Park for Either<A, B>
-where
- A: Park,
- B: Park,
-{
- type Unpark = Either<A::Unpark, B::Unpark>;
- type Error = Either<A::Error, B::Error>;
-
- fn unpark(&self) -> Self::Unpark {
- match self {
- Either::A(a) => Either::A(a.unpark()),
- Either::B(b) => Either::B(b.unpark()),
- }
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- match self {
- Either::A(a) => a.park().map_err(Either::A),
- Either::B(b) => b.park().map_err(Either::B),
- }
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- match self {
- Either::A(a) => a.park_timeout(duration).map_err(Either::A),
- Either::B(b) => b.park_timeout(duration).map_err(Either::B),
- }
- }
-
- fn shutdown(&mut self) {
- match self {
- Either::A(a) => a.shutdown(),
- Either::B(b) => b.shutdown(),
- }
- }
-}
-
-impl<A, B> Unpark for Either<A, B>
-where
- A: Unpark,
- B: Unpark,
-{
- fn unpark(&self) {
- match self {
- Either::A(a) => a.unpark(),
- Either::B(b) => b.unpark(),
- }
- }
-}
-
-impl<A, B> fmt::Debug for Either<A, B>
-where
- A: fmt::Debug,
- B: fmt::Debug,
-{
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- match self {
- Either::A(a) => a.fmt(fmt),
- Either::B(b) => b.fmt(fmt),
- }
- }
-}
diff --git a/src/park/mod.rs b/src/park/mod.rs
deleted file mode 100644
index 87d04ff..0000000
--- a/src/park/mod.rs
+++ /dev/null
@@ -1,117 +0,0 @@
-//! Abstraction over blocking and unblocking the current thread.
-//!
-//! Provides an abstraction over blocking the current thread. This is similar to
-//! the park / unpark constructs provided by `std` but made generic. This allows
-//! embedding custom functionality to perform when the thread is blocked.
-//!
-//! A blocked `Park` instance is unblocked by calling `unpark` on its
-//! `Unpark` handle.
-//!
-//! The `ParkThread` struct implements `Park` using `thread::park` to put the
-//! thread to sleep. The Tokio reactor also implements park, but uses
-//! `mio::Poll` to block the thread instead.
-//!
-//! The `Park` trait is composable. A timer implementation might decorate a
-//! `Park` implementation by checking if any timeouts have elapsed after the
-//! inner `Park` implementation unblocks.
-//!
-//! # Model
-//!
-//! Conceptually, each `Park` instance has an associated token, which is
-//! initially not present:
-//!
-//! * The `park` method blocks the current thread unless or until the token is
-//! available, at which point it atomically consumes the token.
-//! * The `unpark` method atomically makes the token available if it wasn't
-//! already.
-//!
-//! Some things to note:
-//!
-//! * If `unpark` is called before `park`, the next call to `park` will
-//! **not** block the thread.
-//! * **Spurious** wakeups are permitted, i.e., the `park` method may unblock
-//! even if `unpark` was not called.
-//! * `park_timeout` does the same as `park` but allows specifying a maximum
-//! time to block the thread for.
-
-cfg_rt! {
- pub(crate) mod either;
-}
-
-#[cfg(any(feature = "rt", feature = "sync"))]
-pub(crate) mod thread;
-
-use std::fmt::Debug;
-use std::sync::Arc;
-use std::time::Duration;
-
-/// Blocks the current thread.
-pub(crate) trait Park {
- /// Unpark handle type for the `Park` implementation.
- type Unpark: Unpark;
-
- /// Error returned by `park`.
- type Error: Debug;
-
- /// Gets a new `Unpark` handle associated with this `Park` instance.
- fn unpark(&self) -> Self::Unpark;
-
- /// Blocks the current thread unless or until the token is available.
- ///
- /// A call to `park` does not guarantee that the thread will remain blocked
- /// forever, and callers should be prepared for this possibility. This
- /// function may wakeup spuriously for any reason.
- ///
- /// # Panics
- ///
- /// This function **should** not panic, but ultimately, panics are left as
- /// an implementation detail. Refer to the documentation for the specific
- /// `Park` implementation.
- fn park(&mut self) -> Result<(), Self::Error>;
-
- /// Parks the current thread for at most `duration`.
- ///
- /// This function is the same as `park` but allows specifying a maximum time
- /// to block the thread for.
- ///
- /// Same as `park`, there is no guarantee that the thread will remain
- /// blocked for any amount of time. Spurious wakeups are permitted for any
- /// reason.
- ///
- /// # Panics
- ///
- /// This function **should** not panic, but ultimately, panics are left as
- /// an implementation detail. Refer to the documentation for the specific
- /// `Park` implementation.
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error>;
-
- /// Releases all resources holded by the parker for proper leak-free shutdown.
- fn shutdown(&mut self);
-}
-
-/// Unblock a thread blocked by the associated `Park` instance.
-pub(crate) trait Unpark: Sync + Send + 'static {
- /// Unblocks a thread that is blocked by the associated `Park` handle.
- ///
- /// Calling `unpark` atomically makes available the unpark token, if it is
- /// not already available.
- ///
- /// # Panics
- ///
- /// This function **should** not panic, but ultimately, panics are left as
- /// an implementation detail. Refer to the documentation for the specific
- /// `Unpark` implementation.
- fn unpark(&self);
-}
-
-impl Unpark for Box<dyn Unpark> {
- fn unpark(&self) {
- (**self).unpark()
- }
-}
-
-impl Unpark for Arc<dyn Unpark> {
- fn unpark(&self) {
- (**self).unpark()
- }
-}
diff --git a/src/park/thread.rs b/src/park/thread.rs
deleted file mode 100644
index 27ce202..0000000
--- a/src/park/thread.rs
+++ /dev/null
@@ -1,346 +0,0 @@
-#![cfg_attr(not(feature = "full"), allow(dead_code))]
-
-use crate::loom::sync::atomic::AtomicUsize;
-use crate::loom::sync::{Arc, Condvar, Mutex};
-use crate::park::{Park, Unpark};
-
-use std::sync::atomic::Ordering::SeqCst;
-use std::time::Duration;
-
-#[derive(Debug)]
-pub(crate) struct ParkThread {
- inner: Arc<Inner>,
-}
-
-pub(crate) type ParkError = ();
-
-/// Unblocks a thread that was blocked by `ParkThread`.
-#[derive(Clone, Debug)]
-pub(crate) struct UnparkThread {
- inner: Arc<Inner>,
-}
-
-#[derive(Debug)]
-struct Inner {
- state: AtomicUsize,
- mutex: Mutex<()>,
- condvar: Condvar,
-}
-
-const EMPTY: usize = 0;
-const PARKED: usize = 1;
-const NOTIFIED: usize = 2;
-
-thread_local! {
- static CURRENT_PARKER: ParkThread = ParkThread::new();
-}
-
-// ==== impl ParkThread ====
-
-impl ParkThread {
- pub(crate) fn new() -> Self {
- Self {
- inner: Arc::new(Inner {
- state: AtomicUsize::new(EMPTY),
- mutex: Mutex::new(()),
- condvar: Condvar::new(),
- }),
- }
- }
-}
-
-impl Park for ParkThread {
- type Unpark = UnparkThread;
- type Error = ParkError;
-
- fn unpark(&self) -> Self::Unpark {
- let inner = self.inner.clone();
- UnparkThread { inner }
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- self.inner.park();
- Ok(())
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.inner.park_timeout(duration);
- Ok(())
- }
-
- fn shutdown(&mut self) {
- self.inner.shutdown();
- }
-}
-
-// ==== impl Inner ====
-
-impl Inner {
- /// Parks the current thread for at most `dur`.
- fn park(&self) {
- // If we were previously notified then we consume this notification and
- // return quickly.
- if self
- .state
- .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
- .is_ok()
- {
- return;
- }
-
- // Otherwise we need to coordinate going to sleep
- let mut m = self.mutex.lock();
-
- match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) {
- Ok(_) => {}
- Err(NOTIFIED) => {
- // We must read here, even though we know it will be `NOTIFIED`.
- // This is because `unpark` may have been called again since we read
- // `NOTIFIED` in the `compare_exchange` above. We must perform an
- // acquire operation that synchronizes with that `unpark` to observe
- // any writes it made before the call to unpark. To do that we must
- // read from the write it made to `state`.
- let old = self.state.swap(EMPTY, SeqCst);
- debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly");
-
- return;
- }
- Err(actual) => panic!("inconsistent park state; actual = {}", actual),
- }
-
- loop {
- m = self.condvar.wait(m).unwrap();
-
- if self
- .state
- .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
- .is_ok()
- {
- // got a notification
- return;
- }
-
- // spurious wakeup, go back to sleep
- }
- }
-
- fn park_timeout(&self, dur: Duration) {
- // Like `park` above we have a fast path for an already-notified thread,
- // and afterwards we start coordinating for a sleep. Return quickly.
- if self
- .state
- .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
- .is_ok()
- {
- return;
- }
-
- if dur == Duration::from_millis(0) {
- return;
- }
-
- let m = self.mutex.lock();
-
- match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) {
- Ok(_) => {}
- Err(NOTIFIED) => {
- // We must read again here, see `park`.
- let old = self.state.swap(EMPTY, SeqCst);
- debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly");
-
- return;
- }
- Err(actual) => panic!("inconsistent park_timeout state; actual = {}", actual),
- }
-
- // Wait with a timeout, and if we spuriously wake up or otherwise wake up
- // from a notification, we just want to unconditionally set the state back to
- // empty, either consuming a notification or un-flagging ourselves as
- // parked.
- let (_m, _result) = self.condvar.wait_timeout(m, dur).unwrap();
-
- match self.state.swap(EMPTY, SeqCst) {
- NOTIFIED => {} // got a notification, hurray!
- PARKED => {} // no notification, alas
- n => panic!("inconsistent park_timeout state: {}", n),
- }
- }
-
- fn unpark(&self) {
- // To ensure the unparked thread will observe any writes we made before
- // this call, we must perform a release operation that `park` can
- // synchronize with. To do that we must write `NOTIFIED` even if `state`
- // is already `NOTIFIED`. That is why this must be a swap rather than a
- // compare-and-swap that returns if it reads `NOTIFIED` on failure.
- match self.state.swap(NOTIFIED, SeqCst) {
- EMPTY => return, // no one was waiting
- NOTIFIED => return, // already unparked
- PARKED => {} // gotta go wake someone up
- _ => panic!("inconsistent state in unpark"),
- }
-
- // There is a period between when the parked thread sets `state` to
- // `PARKED` (or last checked `state` in the case of a spurious wake
- // up) and when it actually waits on `cvar`. If we were to notify
- // during this period it would be ignored and then when the parked
- // thread went to sleep it would never wake up. Fortunately, it has
- // `lock` locked at this stage so we can acquire `lock` to wait until
- // it is ready to receive the notification.
- //
- // Releasing `lock` before the call to `notify_one` means that when the
- // parked thread wakes it doesn't get woken only to have to wait for us
- // to release `lock`.
- drop(self.mutex.lock());
-
- self.condvar.notify_one()
- }
-
- fn shutdown(&self) {
- self.condvar.notify_all();
- }
-}
-
-impl Default for ParkThread {
- fn default() -> Self {
- Self::new()
- }
-}
-
-// ===== impl UnparkThread =====
-
-impl Unpark for UnparkThread {
- fn unpark(&self) {
- self.inner.unpark();
- }
-}
-
-use std::future::Future;
-use std::marker::PhantomData;
-use std::mem;
-use std::rc::Rc;
-use std::task::{RawWaker, RawWakerVTable, Waker};
-
-/// Blocks the current thread using a condition variable.
-#[derive(Debug)]
-pub(crate) struct CachedParkThread {
- _anchor: PhantomData<Rc<()>>,
-}
-
-impl CachedParkThread {
- /// Creates a new `ParkThread` handle for the current thread.
- ///
- /// This type cannot be moved to other threads, so it should be created on
- /// the thread that the caller intends to park.
- pub(crate) fn new() -> CachedParkThread {
- CachedParkThread {
- _anchor: PhantomData,
- }
- }
-
- pub(crate) fn get_unpark(&self) -> Result<UnparkThread, ParkError> {
- self.with_current(|park_thread| park_thread.unpark())
- }
-
- /// Gets a reference to the `ParkThread` handle for this thread.
- fn with_current<F, R>(&self, f: F) -> Result<R, ParkError>
- where
- F: FnOnce(&ParkThread) -> R,
- {
- CURRENT_PARKER.try_with(|inner| f(inner)).map_err(|_| ())
- }
-
- pub(crate) fn block_on<F: Future>(&mut self, f: F) -> Result<F::Output, ParkError> {
- use std::task::Context;
- use std::task::Poll::Ready;
-
- // `get_unpark()` should not return a Result
- let waker = self.get_unpark()?.into_waker();
- let mut cx = Context::from_waker(&waker);
-
- pin!(f);
-
- loop {
- if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) {
- return Ok(v);
- }
-
- self.park()?;
- }
- }
-}
-
-impl Park for CachedParkThread {
- type Unpark = UnparkThread;
- type Error = ParkError;
-
- fn unpark(&self) -> Self::Unpark {
- self.get_unpark().unwrap()
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- self.with_current(|park_thread| park_thread.inner.park())?;
- Ok(())
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.with_current(|park_thread| park_thread.inner.park_timeout(duration))?;
- Ok(())
- }
-
- fn shutdown(&mut self) {
- let _ = self.with_current(|park_thread| park_thread.inner.shutdown());
- }
-}
-
-impl UnparkThread {
- pub(crate) fn into_waker(self) -> Waker {
- unsafe {
- let raw = unparker_to_raw_waker(self.inner);
- Waker::from_raw(raw)
- }
- }
-}
-
-impl Inner {
- #[allow(clippy::wrong_self_convention)]
- fn into_raw(this: Arc<Inner>) -> *const () {
- Arc::into_raw(this) as *const ()
- }
-
- unsafe fn from_raw(ptr: *const ()) -> Arc<Inner> {
- Arc::from_raw(ptr as *const Inner)
- }
-}
-
-unsafe fn unparker_to_raw_waker(unparker: Arc<Inner>) -> RawWaker {
- RawWaker::new(
- Inner::into_raw(unparker),
- &RawWakerVTable::new(clone, wake, wake_by_ref, drop_waker),
- )
-}
-
-unsafe fn clone(raw: *const ()) -> RawWaker {
- let unparker = Inner::from_raw(raw);
-
- // Increment the ref count
- mem::forget(unparker.clone());
-
- unparker_to_raw_waker(unparker)
-}
-
-unsafe fn drop_waker(raw: *const ()) {
- let _ = Inner::from_raw(raw);
-}
-
-unsafe fn wake(raw: *const ()) {
- let unparker = Inner::from_raw(raw);
- unparker.unpark();
-}
-
-unsafe fn wake_by_ref(raw: *const ()) {
- let unparker = Inner::from_raw(raw);
- unparker.unpark();
-
- // We don't actually own a reference to the unparker
- mem::forget(unparker);
-}
diff --git a/src/process/mod.rs b/src/process/mod.rs
index 6eeefdb..66e4212 100644
--- a/src/process/mod.rs
+++ b/src/process/mod.rs
@@ -97,6 +97,59 @@
//! }
//! ```
//!
+//! Here is another example using `sort` writing into the child process
+//! standard input, capturing the output of the sorted text.
+//!
+//! ```no_run
+//! use tokio::io::AsyncWriteExt;
+//! use tokio::process::Command;
+//!
+//! use std::process::Stdio;
+//!
+//! #[tokio::main]
+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let mut cmd = Command::new("sort");
+//!
+//! // Specifying that we want pipe both the output and the input.
+//! // Similarly to capturing the output, by configuring the pipe
+//! // to stdin it can now be used as an asynchronous writer.
+//! cmd.stdout(Stdio::piped());
+//! cmd.stdin(Stdio::piped());
+//!
+//! let mut child = cmd.spawn().expect("failed to spawn command");
+//!
+//! // These are the animals we want to sort
+//! let animals: &[&str] = &["dog", "bird", "frog", "cat", "fish"];
+//!
+//! let mut stdin = child
+//! .stdin
+//! .take()
+//! .expect("child did not have a handle to stdin");
+//!
+//! // Write our animals to the child process
+//! // Note that the behavior of `sort` is to buffer _all input_ before writing any output.
+//! // In the general sense, it is recommended to write to the child in a separate task as
+//! // awaiting its exit (or output) to avoid deadlocks (for example, the child tries to write
+//! // some output but gets stuck waiting on the parent to read from it, meanwhile the parent
+//! // is stuck waiting to write its input completely before reading the output).
+//! stdin
+//! .write(animals.join("\n").as_bytes())
+//! .await
+//! .expect("could not write to stdin");
+//!
+//! // We drop the handle here which signals EOF to the child process.
+//! // This tells the child process that it there is no more data on the pipe.
+//! drop(stdin);
+//!
+//! let op = child.wait_with_output().await?;
+//!
+//! // Results should come back in sorted order
+//! assert_eq!(op.stdout, "bird\ncat\ndog\nfish\nfrog\n".as_bytes());
+//!
+//! Ok(())
+//! }
+//! ```
+//!
//! With some coordination, we can also pipe the output of one command into
//! another.
//!
@@ -264,6 +317,12 @@
Self::from(StdCommand::new(program))
}
+ /// Cheaply convert to a `&std::process::Command` for places where the type from the standard
+ /// library is expected.
+ pub fn as_std(&self) -> &StdCommand {
+ &self.std
+ }
+
/// Adds an argument to pass to the program.
///
/// Only one argument can be passed per use. So instead of:
@@ -631,6 +690,36 @@
self
}
+ /// Sets the process group ID (PGID) of the child process. Equivalent to a
+ /// setpgid call in the child process, but may be more efficient.
+ ///
+ /// Process groups determine which processes receive signals.
+ ///
+ /// **Note**: This is an [unstable API][unstable] but will be stabilised once
+ /// tokio's MSRV is sufficiently new. See [the documentation on
+ /// unstable features][unstable] for details about using unstable features.
+ ///
+ /// If you want similar behaviour without using this unstable feature you can
+ /// create a [`std::process::Command`] and convert that into a
+ /// [`tokio::process::Command`] using the `From` trait.
+ ///
+ /// [unstable]: crate#unstable-features
+ /// [`tokio::process::Command`]: crate::process::Command
+ ///
+ /// ```no_run
+ /// use tokio::process::Command;
+ ///
+ /// let command = Command::new("ls")
+ /// .process_group(0);
+ /// ```
+ #[cfg(unix)]
+ #[cfg(tokio_unstable)]
+ #[cfg_attr(docsrs, doc(cfg(all(unix, tokio_unstable))))]
+ pub fn process_group(&mut self, pgroup: i32) -> &mut Command {
+ self.std.process_group(pgroup);
+ self
+ }
+
/// Executes the command as a child process, returning a handle to it.
///
/// By default, stdin, stdout and stderr are inherited from the parent.
@@ -865,7 +954,7 @@
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
// Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
let ret = Pin::new(&mut self.inner).poll(cx);
@@ -1124,19 +1213,26 @@
pub async fn wait_with_output(mut self) -> io::Result<Output> {
use crate::future::try_join3;
- async fn read_to_end<A: AsyncRead + Unpin>(io: Option<A>) -> io::Result<Vec<u8>> {
+ async fn read_to_end<A: AsyncRead + Unpin>(io: &mut Option<A>) -> io::Result<Vec<u8>> {
let mut vec = Vec::new();
- if let Some(mut io) = io {
- crate::io::util::read_to_end(&mut io, &mut vec).await?;
+ if let Some(io) = io.as_mut() {
+ crate::io::util::read_to_end(io, &mut vec).await?;
}
Ok(vec)
}
- let stdout_fut = read_to_end(self.stdout.take());
- let stderr_fut = read_to_end(self.stderr.take());
+ let mut stdout_pipe = self.stdout.take();
+ let mut stderr_pipe = self.stderr.take();
+
+ let stdout_fut = read_to_end(&mut stdout_pipe);
+ let stderr_fut = read_to_end(&mut stderr_pipe);
let (status, stdout, stderr) = try_join3(self.wait(), stdout_fut, stderr_fut).await?;
+ // Drop happens after `try_join` due to <https://github.com/tokio-rs/tokio/issues/4309>
+ drop(stdout_pipe);
+ drop(stderr_pipe);
+
Ok(Output {
status,
stdout,
@@ -1219,41 +1315,51 @@
impl AsyncWrite for ChildStdin {
fn poll_write(
- self: Pin<&mut Self>,
+ mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
- self.inner.poll_write(cx, buf)
+ Pin::new(&mut self.inner).poll_write(cx, buf)
}
- fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
- Poll::Ready(Ok(()))
+ fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.inner).poll_flush(cx)
}
- fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
- Poll::Ready(Ok(()))
+ fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.inner).poll_shutdown(cx)
+ }
+
+ fn poll_write_vectored(
+ mut self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ bufs: &[io::IoSlice<'_>],
+ ) -> Poll<Result<usize, io::Error>> {
+ Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
+ }
+
+ fn is_write_vectored(&self) -> bool {
+ self.inner.is_write_vectored()
}
}
impl AsyncRead for ChildStdout {
fn poll_read(
- self: Pin<&mut Self>,
+ mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
- // Safety: pipes support reading into uninitialized memory
- unsafe { self.inner.poll_read(cx, buf) }
+ Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
impl AsyncRead for ChildStderr {
fn poll_read(
- self: Pin<&mut Self>,
+ mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
- // Safety: pipes support reading into uninitialized memory
- unsafe { self.inner.poll_read(cx, buf) }
+ Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
diff --git a/src/process/unix/driver.rs b/src/process/unix/driver.rs
deleted file mode 100644
index 84dc8fb..0000000
--- a/src/process/unix/driver.rs
+++ /dev/null
@@ -1,58 +0,0 @@
-#![cfg_attr(not(feature = "rt"), allow(dead_code))]
-
-//! Process driver.
-
-use crate::park::Park;
-use crate::process::unix::GlobalOrphanQueue;
-use crate::signal::unix::driver::{Driver as SignalDriver, Handle as SignalHandle};
-
-use std::io;
-use std::time::Duration;
-
-/// Responsible for cleaning up orphaned child processes on Unix platforms.
-#[derive(Debug)]
-pub(crate) struct Driver {
- park: SignalDriver,
- signal_handle: SignalHandle,
-}
-
-// ===== impl Driver =====
-
-impl Driver {
- /// Creates a new signal `Driver` instance that delegates wakeups to `park`.
- pub(crate) fn new(park: SignalDriver) -> Self {
- let signal_handle = park.handle();
-
- Self {
- park,
- signal_handle,
- }
- }
-}
-
-// ===== impl Park for Driver =====
-
-impl Park for Driver {
- type Unpark = <SignalDriver as Park>::Unpark;
- type Error = io::Error;
-
- fn unpark(&self) -> Self::Unpark {
- self.park.unpark()
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- self.park.park()?;
- GlobalOrphanQueue::reap_orphans(&self.signal_handle);
- Ok(())
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.park.park_timeout(duration)?;
- GlobalOrphanQueue::reap_orphans(&self.signal_handle);
- Ok(())
- }
-
- fn shutdown(&mut self) {
- self.park.shutdown()
- }
-}
diff --git a/src/process/unix/mod.rs b/src/process/unix/mod.rs
index 576fe6c..78c792c 100644
--- a/src/process/unix/mod.rs
+++ b/src/process/unix/mod.rs
@@ -21,23 +21,20 @@
//! processes in general aren't scalable (e.g. millions) so it shouldn't be that
//! bad in theory...
-pub(crate) mod driver;
-
pub(crate) mod orphan;
use orphan::{OrphanQueue, OrphanQueueImpl, Wait};
mod reap;
use reap::Reaper;
-use crate::io::PollEvented;
+use crate::io::{AsyncRead, AsyncWrite, PollEvented, ReadBuf};
use crate::process::kill::Kill;
use crate::process::SpawnedChild;
-use crate::signal::unix::driver::Handle as SignalHandle;
+use crate::runtime::signal::Handle as SignalHandle;
use crate::signal::unix::{signal, Signal, SignalKind};
use mio::event::Source;
use mio::unix::SourceFd;
-use once_cell::sync::Lazy;
use std::fmt;
use std::fs::File;
use std::future::Future;
@@ -64,25 +61,41 @@
}
}
-static ORPHAN_QUEUE: Lazy<OrphanQueueImpl<StdChild>> = Lazy::new(OrphanQueueImpl::new);
+cfg_not_has_const_mutex_new! {
+ fn get_orphan_queue() -> &'static OrphanQueueImpl<StdChild> {
+ use crate::util::once_cell::OnceCell;
+
+ static ORPHAN_QUEUE: OnceCell<OrphanQueueImpl<StdChild>> = OnceCell::new();
+
+ ORPHAN_QUEUE.get(OrphanQueueImpl::new)
+ }
+}
+
+cfg_has_const_mutex_new! {
+ fn get_orphan_queue() -> &'static OrphanQueueImpl<StdChild> {
+ static ORPHAN_QUEUE: OrphanQueueImpl<StdChild> = OrphanQueueImpl::new();
+
+ &ORPHAN_QUEUE
+ }
+}
pub(crate) struct GlobalOrphanQueue;
impl fmt::Debug for GlobalOrphanQueue {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- ORPHAN_QUEUE.fmt(fmt)
+ get_orphan_queue().fmt(fmt)
}
}
impl GlobalOrphanQueue {
- fn reap_orphans(handle: &SignalHandle) {
- ORPHAN_QUEUE.reap_orphans(handle)
+ pub(crate) fn reap_orphans(handle: &SignalHandle) {
+ get_orphan_queue().reap_orphans(handle)
}
}
impl OrphanQueue<StdChild> for GlobalOrphanQueue {
fn push_orphan(&self, orphan: StdChild) {
- ORPHAN_QUEUE.push_orphan(orphan)
+ get_orphan_queue().push_orphan(orphan)
}
}
@@ -143,7 +156,7 @@
#[derive(Debug)]
pub(crate) struct Pipe {
- // Actually a pipe and not a File. However, we are reusing `File` to get
+ // Actually a pipe is not a File. However, we are reusing `File` to get
// close on drop. This is a similar trick as `mio`.
fd: File,
}
@@ -169,6 +182,10 @@
fn flush(&mut self) -> io::Result<()> {
(&self.fd).flush()
}
+
+ fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result<usize> {
+ (&self.fd).write_vectored(bufs)
+ }
}
impl AsRawFd for Pipe {
@@ -177,8 +194,8 @@
}
}
-pub(crate) fn convert_to_stdio(io: PollEvented<Pipe>) -> io::Result<Stdio> {
- let mut fd = io.into_inner()?.fd;
+pub(crate) fn convert_to_stdio(io: ChildStdio) -> io::Result<Stdio> {
+ let mut fd = io.inner.into_inner()?.fd;
// Ensure that the fd to be inherited is set to *blocking* mode, as this
// is the default that virtually all programs expect to have. Those
@@ -213,7 +230,62 @@
}
}
-pub(crate) type ChildStdio = PollEvented<Pipe>;
+pub(crate) struct ChildStdio {
+ inner: PollEvented<Pipe>,
+}
+
+impl fmt::Debug for ChildStdio {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.inner.fmt(fmt)
+ }
+}
+
+impl AsRawFd for ChildStdio {
+ fn as_raw_fd(&self) -> RawFd {
+ self.inner.as_raw_fd()
+ }
+}
+
+impl AsyncWrite for ChildStdio {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &[u8],
+ ) -> Poll<io::Result<usize>> {
+ self.inner.poll_write(cx, buf)
+ }
+
+ fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn poll_write_vectored(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ bufs: &[io::IoSlice<'_>],
+ ) -> Poll<Result<usize, io::Error>> {
+ self.inner.poll_write_vectored(cx, bufs)
+ }
+
+ fn is_write_vectored(&self) -> bool {
+ true
+ }
+}
+
+impl AsyncRead for ChildStdio {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<io::Result<()>> {
+ // Safety: pipes support reading into uninitialized memory
+ unsafe { self.inner.poll_read(cx, buf) }
+ }
+}
fn set_nonblocking<T: AsRawFd>(fd: &mut T, nonblocking: bool) -> io::Result<()> {
unsafe {
@@ -238,7 +310,7 @@
Ok(())
}
-pub(super) fn stdio<T>(io: T) -> io::Result<PollEvented<Pipe>>
+pub(super) fn stdio<T>(io: T) -> io::Result<ChildStdio>
where
T: IntoRawFd,
{
@@ -246,5 +318,5 @@
let mut pipe = Pipe::from(io);
set_nonblocking(&mut pipe, true)?;
- PollEvented::new(pipe)
+ PollEvented::new(pipe).map(|inner| ChildStdio { inner })
}
diff --git a/src/process/unix/orphan.rs b/src/process/unix/orphan.rs
index 1b0022c..3407196 100644
--- a/src/process/unix/orphan.rs
+++ b/src/process/unix/orphan.rs
@@ -1,5 +1,5 @@
use crate::loom::sync::{Mutex, MutexGuard};
-use crate::signal::unix::driver::Handle as SignalHandle;
+use crate::runtime::signal::Handle as SignalHandle;
use crate::signal::unix::{signal_with_handle, SignalKind};
use crate::sync::watch;
use std::io;
@@ -43,10 +43,21 @@
}
impl<T> OrphanQueueImpl<T> {
- pub(crate) fn new() -> Self {
- Self {
- sigchild: Mutex::new(None),
- queue: Mutex::new(Vec::new()),
+ cfg_not_has_const_mutex_new! {
+ pub(crate) fn new() -> Self {
+ Self {
+ sigchild: Mutex::new(None),
+ queue: Mutex::new(Vec::new()),
+ }
+ }
+ }
+
+ cfg_has_const_mutex_new! {
+ pub(crate) const fn new() -> Self {
+ Self {
+ sigchild: Mutex::const_new(None),
+ queue: Mutex::const_new(Vec::new()),
+ }
}
}
@@ -120,8 +131,8 @@
#[cfg(all(test, not(loom)))]
pub(crate) mod test {
use super::*;
- use crate::io::driver::Driver as IoDriver;
- use crate::signal::unix::driver::{Driver as SignalDriver, Handle as SignalHandle};
+ use crate::runtime::io::Driver as IoDriver;
+ use crate::runtime::signal::{Driver as SignalDriver, Handle as SignalHandle};
use crate::sync::watch;
use std::cell::{Cell, RefCell};
use std::io;
@@ -280,9 +291,11 @@
drop(signal_guard);
}
+ #[cfg_attr(miri, ignore)] // Miri does not support epoll.
#[test]
fn does_not_register_signal_if_queue_empty() {
- let signal_driver = IoDriver::new().and_then(SignalDriver::new).unwrap();
+ let (io_driver, io_handle) = IoDriver::new(1024).unwrap();
+ let signal_driver = SignalDriver::new(io_driver, &io_handle).unwrap();
let handle = signal_driver.handle();
let orphanage = OrphanQueueImpl::new();
diff --git a/src/process/windows.rs b/src/process/windows.rs
index 136d5b0..0e4c7dd 100644
--- a/src/process/windows.rs
+++ b/src/process/windows.rs
@@ -15,29 +15,31 @@
//! `RegisterWaitForSingleObject` and then wait on the other end of the oneshot
//! from then on out.
-use crate::io::PollEvented;
+use crate::io::{blocking::Blocking, AsyncRead, AsyncWrite, ReadBuf};
use crate::process::kill::Kill;
use crate::process::SpawnedChild;
use crate::sync::oneshot;
-use mio::windows::NamedPipe;
use std::fmt;
+use std::fs::File as StdFile;
use std::future::Future;
use std::io;
-use std::os::windows::prelude::{AsRawHandle, FromRawHandle, IntoRawHandle, RawHandle};
+use std::os::windows::prelude::{AsRawHandle, IntoRawHandle, RawHandle};
use std::pin::Pin;
use std::process::Stdio;
use std::process::{Child as StdChild, Command as StdCommand, ExitStatus};
-use std::ptr;
-use std::task::Context;
-use std::task::Poll;
-use winapi::shared::minwindef::{DWORD, FALSE};
-use winapi::um::handleapi::{DuplicateHandle, INVALID_HANDLE_VALUE};
-use winapi::um::processthreadsapi::GetCurrentProcess;
-use winapi::um::threadpoollegacyapiset::UnregisterWaitEx;
-use winapi::um::winbase::{RegisterWaitForSingleObject, INFINITE};
-use winapi::um::winnt::{
- BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, PVOID, WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE,
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use windows_sys::{
+ Win32::Foundation::{
+ DuplicateHandle, BOOLEAN, DUPLICATE_SAME_ACCESS, HANDLE, INVALID_HANDLE_VALUE,
+ },
+ Win32::System::Threading::{
+ GetCurrentProcess, RegisterWaitForSingleObject, UnregisterWaitEx, WT_EXECUTEINWAITTHREAD,
+ WT_EXECUTEONLYONCE,
+ },
+ Win32::System::WindowsProgramming::INFINITE,
};
#[must_use = "futures do nothing unless polled"]
@@ -119,11 +121,11 @@
}
let (tx, rx) = oneshot::channel();
let ptr = Box::into_raw(Box::new(Some(tx)));
- let mut wait_object = ptr::null_mut();
+ let mut wait_object = 0;
let rc = unsafe {
RegisterWaitForSingleObject(
&mut wait_object,
- inner.child.as_raw_handle(),
+ inner.child.as_raw_handle() as _,
Some(callback),
ptr as *mut _,
INFINITE,
@@ -162,37 +164,106 @@
}
}
-unsafe extern "system" fn callback(ptr: PVOID, _timer_fired: BOOLEAN) {
+unsafe extern "system" fn callback(ptr: *mut std::ffi::c_void, _timer_fired: BOOLEAN) {
let complete = &mut *(ptr as *mut Option<oneshot::Sender<()>>);
let _ = complete.take().unwrap().send(());
}
-pub(crate) type ChildStdio = PollEvented<NamedPipe>;
+#[derive(Debug)]
+struct ArcFile(Arc<StdFile>);
-pub(super) fn stdio<T>(io: T) -> io::Result<PollEvented<NamedPipe>>
+impl io::Read for ArcFile {
+ fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
+ (&*self.0).read(bytes)
+ }
+}
+
+impl io::Write for ArcFile {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ (&*self.0).write(bytes)
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ (&*self.0).flush()
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct ChildStdio {
+ // Used for accessing the raw handle, even if the io version is busy
+ raw: Arc<StdFile>,
+ // For doing I/O operations asynchronously
+ io: Blocking<ArcFile>,
+}
+
+impl AsRawHandle for ChildStdio {
+ fn as_raw_handle(&self) -> RawHandle {
+ self.raw.as_raw_handle()
+ }
+}
+
+impl AsyncRead for ChildStdio {
+ fn poll_read(
+ mut self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.io).poll_read(cx, buf)
+ }
+}
+
+impl AsyncWrite for ChildStdio {
+ fn poll_write(
+ mut self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &[u8],
+ ) -> Poll<io::Result<usize>> {
+ Pin::new(&mut self.io).poll_write(cx, buf)
+ }
+
+ fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.io).poll_flush(cx)
+ }
+
+ fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.io).poll_shutdown(cx)
+ }
+}
+
+pub(super) fn stdio<T>(io: T) -> io::Result<ChildStdio>
where
T: IntoRawHandle,
{
- let pipe = unsafe { NamedPipe::from_raw_handle(io.into_raw_handle()) };
- PollEvented::new(pipe)
+ use std::os::windows::prelude::FromRawHandle;
+
+ let raw = Arc::new(unsafe { StdFile::from_raw_handle(io.into_raw_handle()) });
+ let io = Blocking::new(ArcFile(raw.clone()));
+ Ok(ChildStdio { raw, io })
}
-pub(crate) fn convert_to_stdio(io: PollEvented<NamedPipe>) -> io::Result<Stdio> {
- let named_pipe = io.into_inner()?;
+pub(crate) fn convert_to_stdio(child_stdio: ChildStdio) -> io::Result<Stdio> {
+ let ChildStdio { raw, io } = child_stdio;
+ drop(io); // Try to drop the Arc count here
- // Mio does not implement `IntoRawHandle` for `NamedPipe`, so we'll manually
- // duplicate the handle here...
+ Arc::try_unwrap(raw)
+ .or_else(|raw| duplicate_handle(&*raw))
+ .map(Stdio::from)
+}
+
+fn duplicate_handle<T: AsRawHandle>(io: &T) -> io::Result<StdFile> {
+ use std::os::windows::prelude::FromRawHandle;
+
unsafe {
let mut dup_handle = INVALID_HANDLE_VALUE;
let cur_proc = GetCurrentProcess();
let status = DuplicateHandle(
cur_proc,
- named_pipe.as_raw_handle(),
+ io.as_raw_handle() as _,
cur_proc,
&mut dup_handle,
- 0 as DWORD,
- FALSE,
+ 0,
+ 0,
DUPLICATE_SAME_ACCESS,
);
@@ -200,6 +271,6 @@
return Err(io::Error::last_os_error());
}
- Ok(Stdio::from_raw_handle(dup_handle))
+ Ok(StdFile::from_raw_handle(dup_handle as _))
}
}
diff --git a/src/runtime/basic_scheduler.rs b/src/runtime/basic_scheduler.rs
deleted file mode 100644
index 872d0d5..0000000
--- a/src/runtime/basic_scheduler.rs
+++ /dev/null
@@ -1,519 +0,0 @@
-use crate::future::poll_fn;
-use crate::loom::sync::atomic::AtomicBool;
-use crate::loom::sync::Mutex;
-use crate::park::{Park, Unpark};
-use crate::runtime::context::EnterGuard;
-use crate::runtime::stats::{RuntimeStats, WorkerStatsBatcher};
-use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task};
-use crate::runtime::Callback;
-use crate::sync::notify::Notify;
-use crate::util::{waker_ref, Wake, WakerRef};
-
-use std::cell::RefCell;
-use std::collections::VecDeque;
-use std::fmt;
-use std::future::Future;
-use std::sync::atomic::Ordering::{AcqRel, Release};
-use std::sync::Arc;
-use std::task::Poll::{Pending, Ready};
-use std::time::Duration;
-
-/// Executes tasks on the current thread
-pub(crate) struct BasicScheduler<P: Park> {
- /// Inner state guarded by a mutex that is shared
- /// between all `block_on` calls.
- inner: Mutex<Option<Inner<P>>>,
-
- /// Notifier for waking up other threads to steal the
- /// parker.
- notify: Notify,
-
- /// Sendable task spawner
- spawner: Spawner,
-
- /// This is usually None, but right before dropping the BasicScheduler, it
- /// is changed to `Some` with the context being the runtime's own context.
- /// This ensures that any tasks dropped in the `BasicScheduler`s destructor
- /// run in that runtime's context.
- context_guard: Option<EnterGuard>,
-}
-
-/// The inner scheduler that owns the task queue and the main parker P.
-struct Inner<P: Park> {
- /// Scheduler run queue
- ///
- /// When the scheduler is executed, the queue is removed from `self` and
- /// moved into `Context`.
- ///
- /// This indirection is to allow `BasicScheduler` to be `Send`.
- tasks: Option<Tasks>,
-
- /// Sendable task spawner
- spawner: Spawner,
-
- /// Current tick
- tick: u8,
-
- /// Thread park handle
- park: P,
-
- /// Callback for a worker parking itself
- before_park: Option<Callback>,
- /// Callback for a worker unparking itself
- after_unpark: Option<Callback>,
-
- /// Stats batcher
- stats: WorkerStatsBatcher,
-}
-
-#[derive(Clone)]
-pub(crate) struct Spawner {
- shared: Arc<Shared>,
-}
-
-struct Tasks {
- /// Local run queue.
- ///
- /// Tasks notified from the current thread are pushed into this queue.
- queue: VecDeque<task::Notified<Arc<Shared>>>,
-}
-
-/// A remote scheduler entry.
-///
-/// These are filled in by remote threads sending instructions to the scheduler.
-enum RemoteMsg {
- /// A remote thread wants to spawn a task.
- Schedule(task::Notified<Arc<Shared>>),
-}
-
-// Safety: Used correctly, the task header is "thread safe". Ultimately the task
-// is owned by the current thread executor, for which this instruction is being
-// sent.
-unsafe impl Send for RemoteMsg {}
-
-/// Scheduler state shared between threads.
-struct Shared {
- /// Remote run queue. None if the `Runtime` has been dropped.
- queue: Mutex<Option<VecDeque<RemoteMsg>>>,
-
- /// Collection of all active tasks spawned onto this executor.
- owned: OwnedTasks<Arc<Shared>>,
-
- /// Unpark the blocked thread.
- unpark: Box<dyn Unpark>,
-
- /// Indicates whether the blocked on thread was woken.
- woken: AtomicBool,
-
- /// Keeps track of various runtime stats.
- stats: RuntimeStats,
-}
-
-/// Thread-local context.
-struct Context {
- /// Shared scheduler state
- shared: Arc<Shared>,
-
- /// Local queue
- tasks: RefCell<Tasks>,
-}
-
-/// Initial queue capacity.
-const INITIAL_CAPACITY: usize = 64;
-
-/// Max number of tasks to poll per tick.
-#[cfg(loom)]
-const MAX_TASKS_PER_TICK: usize = 4;
-#[cfg(not(loom))]
-const MAX_TASKS_PER_TICK: usize = 61;
-
-/// How often to check the remote queue first.
-const REMOTE_FIRST_INTERVAL: u8 = 31;
-
-// Tracks the current BasicScheduler.
-scoped_thread_local!(static CURRENT: Context);
-
-impl<P: Park> BasicScheduler<P> {
- pub(crate) fn new(
- park: P,
- before_park: Option<Callback>,
- after_unpark: Option<Callback>,
- ) -> BasicScheduler<P> {
- let unpark = Box::new(park.unpark());
-
- let spawner = Spawner {
- shared: Arc::new(Shared {
- queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))),
- owned: OwnedTasks::new(),
- unpark: unpark as Box<dyn Unpark>,
- woken: AtomicBool::new(false),
- stats: RuntimeStats::new(1),
- }),
- };
-
- let inner = Mutex::new(Some(Inner {
- tasks: Some(Tasks {
- queue: VecDeque::with_capacity(INITIAL_CAPACITY),
- }),
- spawner: spawner.clone(),
- tick: 0,
- park,
- before_park,
- after_unpark,
- stats: WorkerStatsBatcher::new(0),
- }));
-
- BasicScheduler {
- inner,
- notify: Notify::new(),
- spawner,
- context_guard: None,
- }
- }
-
- pub(crate) fn spawner(&self) -> &Spawner {
- &self.spawner
- }
-
- pub(crate) fn block_on<F: Future>(&self, future: F) -> F::Output {
- pin!(future);
-
- // Attempt to steal the dedicated parker and block_on the future if we can there,
- // otherwise, lets select on a notification that the parker is available
- // or the future is complete.
- loop {
- if let Some(inner) = &mut self.take_inner() {
- return inner.block_on(future);
- } else {
- let mut enter = crate::runtime::enter(false);
-
- let notified = self.notify.notified();
- pin!(notified);
-
- if let Some(out) = enter
- .block_on(poll_fn(|cx| {
- if notified.as_mut().poll(cx).is_ready() {
- return Ready(None);
- }
-
- if let Ready(out) = future.as_mut().poll(cx) {
- return Ready(Some(out));
- }
-
- Pending
- }))
- .expect("Failed to `Enter::block_on`")
- {
- return out;
- }
- }
- }
- }
-
- fn take_inner(&self) -> Option<InnerGuard<'_, P>> {
- let inner = self.inner.lock().take()?;
-
- Some(InnerGuard {
- inner: Some(inner),
- basic_scheduler: self,
- })
- }
-
- pub(super) fn set_context_guard(&mut self, guard: EnterGuard) {
- self.context_guard = Some(guard);
- }
-}
-
-impl<P: Park> Inner<P> {
- /// Blocks on the provided future and drives the runtime's driver.
- fn block_on<F: Future>(&mut self, future: F) -> F::Output {
- enter(self, |scheduler, context| {
- let _enter = crate::runtime::enter(false);
- let waker = scheduler.spawner.waker_ref();
- let mut cx = std::task::Context::from_waker(&waker);
-
- pin!(future);
-
- 'outer: loop {
- if scheduler.spawner.reset_woken() {
- scheduler.stats.incr_poll_count();
- if let Ready(v) = crate::coop::budget(|| future.as_mut().poll(&mut cx)) {
- return v;
- }
- }
-
- for _ in 0..MAX_TASKS_PER_TICK {
- // Get and increment the current tick
- let tick = scheduler.tick;
- scheduler.tick = scheduler.tick.wrapping_add(1);
-
- let entry = if tick % REMOTE_FIRST_INTERVAL == 0 {
- scheduler.spawner.pop().or_else(|| {
- context
- .tasks
- .borrow_mut()
- .queue
- .pop_front()
- .map(RemoteMsg::Schedule)
- })
- } else {
- context
- .tasks
- .borrow_mut()
- .queue
- .pop_front()
- .map(RemoteMsg::Schedule)
- .or_else(|| scheduler.spawner.pop())
- };
-
- let entry = match entry {
- Some(entry) => entry,
- None => {
- if let Some(f) = &scheduler.before_park {
- f();
- }
- // This check will fail if `before_park` spawns a task for us to run
- // instead of parking the thread
- if context.tasks.borrow_mut().queue.is_empty() {
- // Park until the thread is signaled
- scheduler.stats.about_to_park();
- scheduler.stats.submit(&scheduler.spawner.shared.stats);
- scheduler.park.park().expect("failed to park");
- scheduler.stats.returned_from_park();
- }
- if let Some(f) = &scheduler.after_unpark {
- f();
- }
-
- // Try polling the `block_on` future next
- continue 'outer;
- }
- };
-
- match entry {
- RemoteMsg::Schedule(task) => {
- scheduler.stats.incr_poll_count();
- let task = context.shared.owned.assert_owner(task);
- crate::coop::budget(|| task.run())
- }
- }
- }
-
- // Yield to the park, this drives the timer and pulls any pending
- // I/O events.
- scheduler.stats.submit(&scheduler.spawner.shared.stats);
- scheduler
- .park
- .park_timeout(Duration::from_millis(0))
- .expect("failed to park");
- }
- })
- }
-}
-
-/// Enters the scheduler context. This sets the queue and other necessary
-/// scheduler state in the thread-local.
-fn enter<F, R, P>(scheduler: &mut Inner<P>, f: F) -> R
-where
- F: FnOnce(&mut Inner<P>, &Context) -> R,
- P: Park,
-{
- // Ensures the run queue is placed back in the `BasicScheduler` instance
- // once `block_on` returns.`
- struct Guard<'a, P: Park> {
- context: Option<Context>,
- scheduler: &'a mut Inner<P>,
- }
-
- impl<P: Park> Drop for Guard<'_, P> {
- fn drop(&mut self) {
- let Context { tasks, .. } = self.context.take().expect("context missing");
- self.scheduler.tasks = Some(tasks.into_inner());
- }
- }
-
- // Remove `tasks` from `self` and place it in a `Context`.
- let tasks = scheduler.tasks.take().expect("invalid state");
-
- let guard = Guard {
- context: Some(Context {
- shared: scheduler.spawner.shared.clone(),
- tasks: RefCell::new(tasks),
- }),
- scheduler,
- };
-
- let context = guard.context.as_ref().unwrap();
- let scheduler = &mut *guard.scheduler;
-
- CURRENT.set(context, || f(scheduler, context))
-}
-
-impl<P: Park> Drop for BasicScheduler<P> {
- fn drop(&mut self) {
- // Avoid a double panic if we are currently panicking and
- // the lock may be poisoned.
-
- let mut inner = match self.inner.lock().take() {
- Some(inner) => inner,
- None if std::thread::panicking() => return,
- None => panic!("Oh no! We never placed the Inner state back, this is a bug!"),
- };
-
- enter(&mut inner, |scheduler, context| {
- // Drain the OwnedTasks collection. This call also closes the
- // collection, ensuring that no tasks are ever pushed after this
- // call returns.
- context.shared.owned.close_and_shutdown_all();
-
- // Drain local queue
- // We already shut down every task, so we just need to drop the task.
- for task in context.tasks.borrow_mut().queue.drain(..) {
- drop(task);
- }
-
- // Drain remote queue and set it to None
- let remote_queue = scheduler.spawner.shared.queue.lock().take();
-
- // Using `Option::take` to replace the shared queue with `None`.
- // We already shut down every task, so we just need to drop the task.
- if let Some(remote_queue) = remote_queue {
- for entry in remote_queue {
- match entry {
- RemoteMsg::Schedule(task) => {
- drop(task);
- }
- }
- }
- }
-
- assert!(context.shared.owned.is_empty());
- });
- }
-}
-
-impl<P: Park> fmt::Debug for BasicScheduler<P> {
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt.debug_struct("BasicScheduler").finish()
- }
-}
-
-// ===== impl Spawner =====
-
-impl Spawner {
- /// Spawns a future onto the basic scheduler
- pub(crate) fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
- where
- F: crate::future::Future + Send + 'static,
- F::Output: Send + 'static,
- {
- let (handle, notified) = self.shared.owned.bind(future, self.shared.clone());
-
- if let Some(notified) = notified {
- self.shared.schedule(notified);
- }
-
- handle
- }
-
- pub(crate) fn stats(&self) -> &RuntimeStats {
- &self.shared.stats
- }
-
- fn pop(&self) -> Option<RemoteMsg> {
- match self.shared.queue.lock().as_mut() {
- Some(queue) => queue.pop_front(),
- None => None,
- }
- }
-
- fn waker_ref(&self) -> WakerRef<'_> {
- // Set woken to true when enter block_on, ensure outer future
- // be polled for the first time when enter loop
- self.shared.woken.store(true, Release);
- waker_ref(&self.shared)
- }
-
- // reset woken to false and return original value
- pub(crate) fn reset_woken(&self) -> bool {
- self.shared.woken.swap(false, AcqRel)
- }
-}
-
-impl fmt::Debug for Spawner {
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt.debug_struct("Spawner").finish()
- }
-}
-
-// ===== impl Shared =====
-
-impl Schedule for Arc<Shared> {
- fn release(&self, task: &Task<Self>) -> Option<Task<Self>> {
- self.owned.remove(task)
- }
-
- fn schedule(&self, task: task::Notified<Self>) {
- CURRENT.with(|maybe_cx| match maybe_cx {
- Some(cx) if Arc::ptr_eq(self, &cx.shared) => {
- cx.tasks.borrow_mut().queue.push_back(task);
- }
- _ => {
- // If the queue is None, then the runtime has shut down. We
- // don't need to do anything with the notification in that case.
- let mut guard = self.queue.lock();
- if let Some(queue) = guard.as_mut() {
- queue.push_back(RemoteMsg::Schedule(task));
- drop(guard);
- self.unpark.unpark();
- }
- }
- });
- }
-}
-
-impl Wake for Shared {
- fn wake(self: Arc<Self>) {
- Wake::wake_by_ref(&self)
- }
-
- /// Wake by reference
- fn wake_by_ref(arc_self: &Arc<Self>) {
- arc_self.woken.store(true, Release);
- arc_self.unpark.unpark();
- }
-}
-
-// ===== InnerGuard =====
-
-/// Used to ensure we always place the Inner value
-/// back into its slot in `BasicScheduler`, even if the
-/// future panics.
-struct InnerGuard<'a, P: Park> {
- inner: Option<Inner<P>>,
- basic_scheduler: &'a BasicScheduler<P>,
-}
-
-impl<P: Park> InnerGuard<'_, P> {
- fn block_on<F: Future>(&mut self, future: F) -> F::Output {
- // The only time inner gets set to `None` is if we have dropped
- // already so this unwrap is safe.
- self.inner.as_mut().unwrap().block_on(future)
- }
-}
-
-impl<P: Park> Drop for InnerGuard<'_, P> {
- fn drop(&mut self) {
- if let Some(scheduler) = self.inner.take() {
- let mut lock = self.basic_scheduler.inner.lock();
-
- // Replace old scheduler back into the state to allow
- // other threads to pick it up and drive it.
- lock.replace(scheduler);
-
- // Wake up other possible threads that could steal
- // the dedicated parker P.
- self.basic_scheduler.notify.notify_one()
- }
- }
-}
diff --git a/src/runtime/blocking/mod.rs b/src/runtime/blocking/mod.rs
index 670ec3a..c42924b 100644
--- a/src/runtime/blocking/mod.rs
+++ b/src/runtime/blocking/mod.rs
@@ -6,10 +6,17 @@
mod pool;
pub(crate) use pool::{spawn_blocking, BlockingPool, Spawner};
+cfg_fs! {
+ pub(crate) use pool::spawn_mandatory_blocking;
+}
+
+cfg_trace! {
+ pub(crate) use pool::Mandatory;
+}
+
mod schedule;
mod shutdown;
mod task;
-pub(crate) use schedule::NoopSchedule;
pub(crate) use task::BlockingTask;
use crate::runtime::Builder;
@@ -17,28 +24,3 @@
pub(crate) fn create_blocking_pool(builder: &Builder, thread_cap: usize) -> BlockingPool {
BlockingPool::new(builder, thread_cap)
}
-
-/*
-cfg_not_blocking_impl! {
- use crate::runtime::Builder;
- use std::time::Duration;
-
- #[derive(Debug, Clone)]
- pub(crate) struct BlockingPool {}
-
- pub(crate) use BlockingPool as Spawner;
-
- pub(crate) fn create_blocking_pool(_builder: &Builder, _thread_cap: usize) -> BlockingPool {
- BlockingPool {}
- }
-
- impl BlockingPool {
- pub(crate) fn spawner(&self) -> &BlockingPool {
- self
- }
-
- pub(crate) fn shutdown(&mut self, _duration: Option<Duration>) {
- }
- }
-}
-*/
diff --git a/src/runtime/blocking/pool.rs b/src/runtime/blocking/pool.rs
index 77ab495..e9f6b66 100644
--- a/src/runtime/blocking/pool.rs
+++ b/src/runtime/blocking/pool.rs
@@ -2,15 +2,16 @@
use crate::loom::sync::{Arc, Condvar, Mutex};
use crate::loom::thread;
-use crate::runtime::blocking::schedule::NoopSchedule;
-use crate::runtime::blocking::shutdown;
+use crate::runtime::blocking::schedule::BlockingSchedule;
+use crate::runtime::blocking::{shutdown, BlockingTask};
use crate::runtime::builder::ThreadNameFn;
-use crate::runtime::context;
use crate::runtime::task::{self, JoinHandle};
use crate::runtime::{Builder, Callback, Handle};
use std::collections::{HashMap, VecDeque};
use std::fmt;
+use std::io;
+use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
pub(crate) struct BlockingPool {
@@ -23,6 +24,53 @@
inner: Arc<Inner>,
}
+#[derive(Default)]
+pub(crate) struct SpawnerMetrics {
+ num_threads: AtomicUsize,
+ num_idle_threads: AtomicUsize,
+ queue_depth: AtomicUsize,
+}
+
+impl SpawnerMetrics {
+ fn num_threads(&self) -> usize {
+ self.num_threads.load(Ordering::Relaxed)
+ }
+
+ fn num_idle_threads(&self) -> usize {
+ self.num_idle_threads.load(Ordering::Relaxed)
+ }
+
+ cfg_metrics! {
+ fn queue_depth(&self) -> usize {
+ self.queue_depth.load(Ordering::Relaxed)
+ }
+ }
+
+ fn inc_num_threads(&self) {
+ self.num_threads.fetch_add(1, Ordering::Relaxed);
+ }
+
+ fn dec_num_threads(&self) {
+ self.num_threads.fetch_sub(1, Ordering::Relaxed);
+ }
+
+ fn inc_num_idle_threads(&self) {
+ self.num_idle_threads.fetch_add(1, Ordering::Relaxed);
+ }
+
+ fn dec_num_idle_threads(&self) -> usize {
+ self.num_idle_threads.fetch_sub(1, Ordering::Relaxed)
+ }
+
+ fn inc_queue_depth(&self) {
+ self.queue_depth.fetch_add(1, Ordering::Relaxed);
+ }
+
+ fn dec_queue_depth(&self) {
+ self.queue_depth.fetch_sub(1, Ordering::Relaxed);
+ }
+}
+
struct Inner {
/// State shared between worker threads.
shared: Mutex<Shared>,
@@ -47,12 +95,13 @@
// Customizable wait timeout.
keep_alive: Duration,
+
+ // Metrics about the pool.
+ metrics: SpawnerMetrics,
}
struct Shared {
queue: VecDeque<Task>,
- num_th: usize,
- num_idle: u32,
num_notify: u32,
shutdown: bool,
shutdown_tx: Option<shutdown::Sender>,
@@ -70,20 +119,89 @@
worker_thread_index: usize,
}
-type Task = task::UnownedTask<NoopSchedule>;
+pub(crate) struct Task {
+ task: task::UnownedTask<BlockingSchedule>,
+ mandatory: Mandatory,
+}
+
+#[derive(PartialEq, Eq)]
+pub(crate) enum Mandatory {
+ #[cfg_attr(not(fs), allow(dead_code))]
+ Mandatory,
+ NonMandatory,
+}
+
+pub(crate) enum SpawnError {
+ /// Pool is shutting down and the task was not scheduled
+ ShuttingDown,
+ /// There are no worker threads available to take the task
+ /// and the OS failed to spawn a new one
+ NoThreads(io::Error),
+}
+
+impl From<SpawnError> for io::Error {
+ fn from(e: SpawnError) -> Self {
+ match e {
+ SpawnError::ShuttingDown => {
+ io::Error::new(io::ErrorKind::Other, "blocking pool shutting down")
+ }
+ SpawnError::NoThreads(e) => e,
+ }
+ }
+}
+
+impl Task {
+ pub(crate) fn new(task: task::UnownedTask<BlockingSchedule>, mandatory: Mandatory) -> Task {
+ Task { task, mandatory }
+ }
+
+ fn run(self) {
+ self.task.run();
+ }
+
+ fn shutdown_or_run_if_mandatory(self) {
+ match self.mandatory {
+ Mandatory::NonMandatory => self.task.shutdown(),
+ Mandatory::Mandatory => self.task.run(),
+ }
+ }
+}
const KEEP_ALIVE: Duration = Duration::from_secs(10);
/// Runs the provided function on an executor dedicated to blocking operations.
+/// Tasks will be scheduled as non-mandatory, meaning they may not get executed
+/// in case of runtime shutdown.
+#[track_caller]
+#[cfg_attr(tokio_wasi, allow(dead_code))]
pub(crate) fn spawn_blocking<F, R>(func: F) -> JoinHandle<R>
where
F: FnOnce() -> R + Send + 'static,
R: Send + 'static,
{
- let rt = context::current();
+ let rt = Handle::current();
rt.spawn_blocking(func)
}
+cfg_fs! {
+ #[cfg_attr(any(
+ all(loom, not(test)), // the function is covered by loom tests
+ test
+ ), allow(dead_code))]
+ /// Runs the provided function on an executor dedicated to blocking
+ /// operations. Tasks will be scheduled as mandatory, meaning they are
+ /// guaranteed to run unless a shutdown is already taking place. In case a
+ /// shutdown is already taking place, `None` will be returned.
+ pub(crate) fn spawn_mandatory_blocking<F, R>(func: F) -> Option<JoinHandle<R>>
+ where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+ {
+ let rt = Handle::current();
+ rt.inner.blocking_spawner().spawn_mandatory_blocking(&rt, func)
+ }
+}
+
// ===== impl BlockingPool =====
impl BlockingPool {
@@ -96,8 +214,6 @@
inner: Arc::new(Inner {
shared: Mutex::new(Shared {
queue: VecDeque::new(),
- num_th: 0,
- num_idle: 0,
num_notify: 0,
shutdown: false,
shutdown_tx: Some(shutdown_tx),
@@ -112,6 +228,7 @@
before_stop: builder.before_stop.clone(),
thread_cap,
keep_alive,
+ metrics: Default::default(),
}),
},
shutdown_rx,
@@ -171,53 +288,162 @@
// ===== impl Spawner =====
impl Spawner {
- pub(crate) fn spawn(&self, task: Task, rt: &Handle) -> Result<(), ()> {
- let shutdown_tx = {
- let mut shared = self.inner.shared.lock();
-
- if shared.shutdown {
- // Shutdown the task
- task.shutdown();
-
- // no need to even push this task; it would never get picked up
- return Err(());
- }
-
- shared.queue.push_back(task);
-
- if shared.num_idle == 0 {
- // No threads are able to process the task.
-
- if shared.num_th == self.inner.thread_cap {
- // At max number of threads
- None
- } else {
- shared.num_th += 1;
- assert!(shared.shutdown_tx.is_some());
- shared.shutdown_tx.clone()
- }
+ #[track_caller]
+ pub(crate) fn spawn_blocking<F, R>(&self, rt: &Handle, func: F) -> JoinHandle<R>
+ where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+ {
+ let (join_handle, spawn_result) =
+ if cfg!(debug_assertions) && std::mem::size_of::<F>() > 2048 {
+ self.spawn_blocking_inner(Box::new(func), Mandatory::NonMandatory, None, rt)
} else {
- // Notify an idle worker thread. The notification counter
- // is used to count the needed amount of notifications
- // exactly. Thread libraries may generate spurious
- // wakeups, this counter is used to keep us in a
- // consistent state.
- shared.num_idle -= 1;
- shared.num_notify += 1;
- self.inner.condvar.notify_one();
+ self.spawn_blocking_inner(func, Mandatory::NonMandatory, None, rt)
+ };
+
+ match spawn_result {
+ Ok(()) => join_handle,
+ // Compat: do not panic here, return the join_handle even though it will never resolve
+ Err(SpawnError::ShuttingDown) => join_handle,
+ Err(SpawnError::NoThreads(e)) => {
+ panic!("OS can't spawn worker thread: {}", e)
+ }
+ }
+ }
+
+ cfg_fs! {
+ #[track_caller]
+ #[cfg_attr(any(
+ all(loom, not(test)), // the function is covered by loom tests
+ test
+ ), allow(dead_code))]
+ pub(crate) fn spawn_mandatory_blocking<F, R>(&self, rt: &Handle, func: F) -> Option<JoinHandle<R>>
+ where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+ {
+ let (join_handle, spawn_result) = if cfg!(debug_assertions) && std::mem::size_of::<F>() > 2048 {
+ self.spawn_blocking_inner(
+ Box::new(func),
+ Mandatory::Mandatory,
+ None,
+ rt,
+ )
+ } else {
+ self.spawn_blocking_inner(
+ func,
+ Mandatory::Mandatory,
+ None,
+ rt,
+ )
+ };
+
+ if spawn_result.is_ok() {
+ Some(join_handle)
+ } else {
None
}
+ }
+ }
+
+ #[track_caller]
+ pub(crate) fn spawn_blocking_inner<F, R>(
+ &self,
+ func: F,
+ is_mandatory: Mandatory,
+ name: Option<&str>,
+ rt: &Handle,
+ ) -> (JoinHandle<R>, Result<(), SpawnError>)
+ where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+ {
+ let fut = BlockingTask::new(func);
+ let id = task::Id::next();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let fut = {
+ use tracing::Instrument;
+ let location = std::panic::Location::caller();
+ let span = tracing::trace_span!(
+ target: "tokio::task::blocking",
+ "runtime.spawn",
+ kind = %"blocking",
+ task.name = %name.unwrap_or_default(),
+ task.id = id.as_u64(),
+ "fn" = %std::any::type_name::<F>(),
+ spawn.location = %format_args!("{}:{}:{}", location.file(), location.line(), location.column()),
+ );
+ fut.instrument(span)
};
- if let Some(shutdown_tx) = shutdown_tx {
- let mut shared = self.inner.shared.lock();
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let _ = name;
- let id = shared.worker_thread_index;
- shared.worker_thread_index += 1;
+ let (task, handle) = task::unowned(fut, BlockingSchedule::new(rt), id);
- let handle = self.spawn_thread(shutdown_tx, rt, id);
+ let spawned = self.spawn_task(Task::new(task, is_mandatory), rt);
+ (handle, spawned)
+ }
- shared.worker_threads.insert(id, handle);
+ fn spawn_task(&self, task: Task, rt: &Handle) -> Result<(), SpawnError> {
+ let mut shared = self.inner.shared.lock();
+
+ if shared.shutdown {
+ // Shutdown the task: it's fine to shutdown this task (even if
+ // mandatory) because it was scheduled after the shutdown of the
+ // runtime began.
+ task.task.shutdown();
+
+ // no need to even push this task; it would never get picked up
+ return Err(SpawnError::ShuttingDown);
+ }
+
+ shared.queue.push_back(task);
+ self.inner.metrics.inc_queue_depth();
+
+ if self.inner.metrics.num_idle_threads() == 0 {
+ // No threads are able to process the task.
+
+ if self.inner.metrics.num_threads() == self.inner.thread_cap {
+ // At max number of threads
+ } else {
+ assert!(shared.shutdown_tx.is_some());
+ let shutdown_tx = shared.shutdown_tx.clone();
+
+ if let Some(shutdown_tx) = shutdown_tx {
+ let id = shared.worker_thread_index;
+
+ match self.spawn_thread(shutdown_tx, rt, id) {
+ Ok(handle) => {
+ self.inner.metrics.inc_num_threads();
+ shared.worker_thread_index += 1;
+ shared.worker_threads.insert(id, handle);
+ }
+ Err(ref e)
+ if is_temporary_os_thread_error(e)
+ && self.inner.metrics.num_threads() > 0 =>
+ {
+ // OS temporarily failed to spawn a new thread.
+ // The task will be picked up eventually by a currently
+ // busy thread.
+ }
+ Err(e) => {
+ // The OS refused to spawn the thread and there is no thread
+ // to pick up the task that has just been pushed to the queue.
+ return Err(SpawnError::NoThreads(e));
+ }
+ }
+ }
+ }
+ } else {
+ // Notify an idle worker thread. The notification counter
+ // is used to count the needed amount of notifications
+ // exactly. Thread libraries may generate spurious
+ // wakeups, this counter is used to keep us in a
+ // consistent state.
+ self.inner.metrics.dec_num_idle_threads();
+ shared.num_notify += 1;
+ self.inner.condvar.notify_one();
}
Ok(())
@@ -228,7 +454,7 @@
shutdown_tx: shutdown::Sender,
rt: &Handle,
id: usize,
- ) -> thread::JoinHandle<()> {
+ ) -> std::io::Result<thread::JoinHandle<()>> {
let mut builder = thread::Builder::new().name((self.inner.thread_name)());
if let Some(stack_size) = self.inner.stack_size {
@@ -237,17 +463,37 @@
let rt = rt.clone();
- builder
- .spawn(move || {
- // Only the reference should be moved into the closure
- let _enter = crate::runtime::context::enter(rt.clone());
- rt.blocking_spawner.inner.run(id);
- drop(shutdown_tx);
- })
- .unwrap()
+ builder.spawn(move || {
+ // Only the reference should be moved into the closure
+ let _enter = rt.enter();
+ rt.inner.blocking_spawner().inner.run(id);
+ drop(shutdown_tx);
+ })
}
}
+cfg_metrics! {
+ impl Spawner {
+ pub(crate) fn num_threads(&self) -> usize {
+ self.inner.metrics.num_threads()
+ }
+
+ pub(crate) fn num_idle_threads(&self) -> usize {
+ self.inner.metrics.num_idle_threads()
+ }
+
+ pub(crate) fn queue_depth(&self) -> usize {
+ self.inner.metrics.queue_depth()
+ }
+ }
+}
+
+// Tells whether the error when spawning a thread is temporary.
+#[inline]
+fn is_temporary_os_thread_error(error: &std::io::Error) -> bool {
+ matches!(error.kind(), std::io::ErrorKind::WouldBlock)
+}
+
impl Inner {
fn run(&self, worker_thread_id: usize) {
if let Some(f) = &self.after_start {
@@ -260,6 +506,7 @@
'main: loop {
// BUSY
while let Some(task) = shared.queue.pop_front() {
+ self.metrics.dec_queue_depth();
drop(shared);
task.run();
@@ -267,7 +514,7 @@
}
// IDLE
- shared.num_idle += 1;
+ self.metrics.inc_num_idle_threads();
while !shared.shutdown {
let lock_result = self.condvar.wait_timeout(shared, self.keep_alive).unwrap();
@@ -301,8 +548,10 @@
if shared.shutdown {
// Drain the queue
while let Some(task) = shared.queue.pop_front() {
+ self.metrics.dec_queue_depth();
drop(shared);
- task.shutdown();
+
+ task.shutdown_or_run_if_mandatory();
shared = self.shared.lock();
}
@@ -310,7 +559,7 @@
// Work was produced, and we "took" it (by decrementing num_notify).
// This means that num_idle was decremented once for our wakeup.
// But, since we are exiting, we need to "undo" that, as we'll stay idle.
- shared.num_idle += 1;
+ self.metrics.inc_num_idle_threads();
// NOTE: Technically we should also do num_notify++ and notify again,
// but since we're shutting down anyway, that won't be necessary.
break;
@@ -318,17 +567,17 @@
}
// Thread exit
- shared.num_th -= 1;
+ self.metrics.dec_num_threads();
// num_idle should now be tracked exactly, panic
// with a descriptive message if it is not the
// case.
- shared.num_idle = shared
- .num_idle
- .checked_sub(1)
- .expect("num_idle underflowed on thread exit");
+ let prev_idle = self.metrics.dec_num_idle_threads();
+ if prev_idle < self.metrics.num_idle_threads() {
+ panic!("num_idle_threads underflowed on thread exit")
+ }
- if shared.shutdown && shared.num_th == 0 {
+ if shared.shutdown && self.metrics.num_threads() == 0 {
self.condvar.notify_one();
}
diff --git a/src/runtime/blocking/schedule.rs b/src/runtime/blocking/schedule.rs
index 5425224..edf775b 100644
--- a/src/runtime/blocking/schedule.rs
+++ b/src/runtime/blocking/schedule.rs
@@ -1,15 +1,52 @@
+#[cfg(feature = "test-util")]
+use crate::runtime::scheduler;
use crate::runtime::task::{self, Task};
+use crate::runtime::Handle;
-/// `task::Schedule` implementation that does nothing. This is unique to the
-/// blocking scheduler as tasks scheduled are not really futures but blocking
-/// operations.
+/// `task::Schedule` implementation that does nothing (except some bookkeeping
+/// in test-util builds). This is unique to the blocking scheduler as tasks
+/// scheduled are not really futures but blocking operations.
///
/// We avoid storing the task by forgetting it in `bind` and re-materializing it
-/// in `release.
-pub(crate) struct NoopSchedule;
+/// in `release`.
+pub(crate) struct BlockingSchedule {
+ #[cfg(feature = "test-util")]
+ handle: Handle,
+}
-impl task::Schedule for NoopSchedule {
+impl BlockingSchedule {
+ #[cfg_attr(not(feature = "test-util"), allow(unused_variables))]
+ pub(crate) fn new(handle: &Handle) -> Self {
+ #[cfg(feature = "test-util")]
+ {
+ match &handle.inner {
+ scheduler::Handle::CurrentThread(handle) => {
+ handle.driver.clock.inhibit_auto_advance();
+ }
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ scheduler::Handle::MultiThread(_) => {}
+ }
+ }
+ BlockingSchedule {
+ #[cfg(feature = "test-util")]
+ handle: handle.clone(),
+ }
+ }
+}
+
+impl task::Schedule for BlockingSchedule {
fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> {
+ #[cfg(feature = "test-util")]
+ {
+ match &self.handle.inner {
+ scheduler::Handle::CurrentThread(handle) => {
+ handle.driver.clock.allow_auto_advance();
+ handle.driver.unpark();
+ }
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ scheduler::Handle::MultiThread(_) => {}
+ }
+ }
None
}
diff --git a/src/runtime/blocking/shutdown.rs b/src/runtime/blocking/shutdown.rs
index e6f4674..fe5abae 100644
--- a/src/runtime/blocking/shutdown.rs
+++ b/src/runtime/blocking/shutdown.rs
@@ -35,13 +35,13 @@
///
/// If the timeout has elapsed, it returns `false`, otherwise it returns `true`.
pub(crate) fn wait(&mut self, timeout: Option<Duration>) -> bool {
- use crate::runtime::enter::try_enter;
+ use crate::runtime::context::try_enter_blocking_region;
if timeout == Some(Duration::from_nanos(0)) {
return false;
}
- let mut e = match try_enter(false) {
+ let mut e = match try_enter_blocking_region() {
Some(enter) => enter,
_ => {
if std::thread::panicking() {
diff --git a/src/runtime/blocking/task.rs b/src/runtime/blocking/task.rs
index 0b7803a..c446175 100644
--- a/src/runtime/blocking/task.rs
+++ b/src/runtime/blocking/task.rs
@@ -37,7 +37,7 @@
// currently goes through Task::poll(), and so is subject to budgeting. That isn't really
// what we want; a blocking task may itself want to run tasks (it might be a Worker!), so
// we want it to start without any budgeting.
- crate::coop::stop();
+ crate::runtime::coop::stop();
Poll::Ready(func())
}
diff --git a/src/runtime/builder.rs b/src/runtime/builder.rs
index 91c365f..ea0df2e 100644
--- a/src/runtime/builder.rs
+++ b/src/runtime/builder.rs
@@ -1,5 +1,6 @@
use crate::runtime::handle::Handle;
-use crate::runtime::{blocking, driver, Callback, Runtime, Spawner};
+use crate::runtime::{blocking, driver, Callback, Runtime};
+use crate::util::rand::{RngSeed, RngSeedGenerator};
use std::fmt;
use std::io;
@@ -43,6 +44,7 @@
/// Whether or not to enable the I/O driver
enable_io: bool,
+ nevents: usize,
/// Whether or not to enable the time driver
enable_time: bool,
@@ -78,13 +80,112 @@
/// Customizable keep alive timeout for BlockingPool
pub(super) keep_alive: Option<Duration>,
+
+ /// How many ticks before pulling a task from the global/remote queue?
+ pub(super) global_queue_interval: u32,
+
+ /// How many ticks before yielding to the driver for timer and I/O events?
+ pub(super) event_interval: u32,
+
+ /// When true, the multi-threade scheduler LIFO slot should not be used.
+ ///
+ /// This option should only be exposed as unstable.
+ pub(super) disable_lifo_slot: bool,
+
+ /// Specify a random number generator seed to provide deterministic results
+ pub(super) seed_generator: RngSeedGenerator,
+
+ #[cfg(tokio_unstable)]
+ pub(super) unhandled_panic: UnhandledPanic,
+}
+
+cfg_unstable! {
+ /// How the runtime should respond to unhandled panics.
+ ///
+ /// Instances of `UnhandledPanic` are passed to `Builder::unhandled_panic`
+ /// to configure the runtime behavior when a spawned task panics.
+ ///
+ /// See [`Builder::unhandled_panic`] for more details.
+ #[derive(Debug, Clone)]
+ #[non_exhaustive]
+ pub enum UnhandledPanic {
+ /// The runtime should ignore panics on spawned tasks.
+ ///
+ /// The panic is forwarded to the task's [`JoinHandle`] and all spawned
+ /// tasks continue running normally.
+ ///
+ /// This is the default behavior.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::{self, UnhandledPanic};
+ ///
+ /// # pub fn main() {
+ /// let rt = runtime::Builder::new_current_thread()
+ /// .unhandled_panic(UnhandledPanic::Ignore)
+ /// .build()
+ /// .unwrap();
+ ///
+ /// let task1 = rt.spawn(async { panic!("boom"); });
+ /// let task2 = rt.spawn(async {
+ /// // This task completes normally
+ /// "done"
+ /// });
+ ///
+ /// rt.block_on(async {
+ /// // The panic on the first task is forwarded to the `JoinHandle`
+ /// assert!(task1.await.is_err());
+ ///
+ /// // The second task completes normally
+ /// assert!(task2.await.is_ok());
+ /// })
+ /// # }
+ /// ```
+ ///
+ /// [`JoinHandle`]: struct@crate::task::JoinHandle
+ Ignore,
+
+ /// The runtime should immediately shutdown if a spawned task panics.
+ ///
+ /// The runtime will immediately shutdown even if the panicked task's
+ /// [`JoinHandle`] is still available. All further spawned tasks will be
+ /// immediately dropped and call to [`Runtime::block_on`] will panic.
+ ///
+ /// # Examples
+ ///
+ /// ```should_panic
+ /// use tokio::runtime::{self, UnhandledPanic};
+ ///
+ /// # pub fn main() {
+ /// let rt = runtime::Builder::new_current_thread()
+ /// .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ /// .build()
+ /// .unwrap();
+ ///
+ /// rt.spawn(async { panic!("boom"); });
+ /// rt.spawn(async {
+ /// // This task never completes.
+ /// });
+ ///
+ /// rt.block_on(async {
+ /// // Do some work
+ /// # loop { tokio::task::yield_now().await; }
+ /// })
+ /// # }
+ /// ```
+ ///
+ /// [`JoinHandle`]: struct@crate::task::JoinHandle
+ ShutdownRuntime,
+ }
}
pub(crate) type ThreadNameFn = std::sync::Arc<dyn Fn() -> String + Send + Sync + 'static>;
+#[derive(Clone, Copy)]
pub(crate) enum Kind {
CurrentThread,
- #[cfg(feature = "rt-multi-thread")]
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
MultiThread,
}
@@ -98,28 +199,38 @@
///
/// [`LocalSet`]: crate::task::LocalSet
pub fn new_current_thread() -> Builder {
- Builder::new(Kind::CurrentThread)
+ #[cfg(loom)]
+ const EVENT_INTERVAL: u32 = 4;
+ // The number `61` is fairly arbitrary. I believe this value was copied from golang.
+ #[cfg(not(loom))]
+ const EVENT_INTERVAL: u32 = 61;
+
+ Builder::new(Kind::CurrentThread, 31, EVENT_INTERVAL)
}
- /// Returns a new builder with the multi thread scheduler selected.
- ///
- /// Configuration methods can be chained on the return value.
- #[cfg(feature = "rt-multi-thread")]
- #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
- pub fn new_multi_thread() -> Builder {
- Builder::new(Kind::MultiThread)
+ cfg_not_wasi! {
+ /// Returns a new builder with the multi thread scheduler selected.
+ ///
+ /// Configuration methods can be chained on the return value.
+ #[cfg(feature = "rt-multi-thread")]
+ #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
+ pub fn new_multi_thread() -> Builder {
+ // The number `61` is fairly arbitrary. I believe this value was copied from golang.
+ Builder::new(Kind::MultiThread, 61, 61)
+ }
}
/// Returns a new runtime builder initialized with default configuration
/// values.
///
/// Configuration methods can be chained on the return value.
- pub(crate) fn new(kind: Kind) -> Builder {
+ pub(crate) fn new(kind: Kind, global_queue_interval: u32, event_interval: u32) -> Builder {
Builder {
kind,
// I/O defaults to "off"
enable_io: false,
+ nevents: 1024,
// Time defaults to "off"
enable_time: false,
@@ -127,6 +238,7 @@
// The clock starts not-paused
start_paused: false,
+ // Read from environment variable first in multi-threaded mode.
// Default to lazy auto-detection (one thread per CPU core)
worker_threads: None,
@@ -145,6 +257,18 @@
after_unpark: None,
keep_alive: None,
+
+ // Defaults for these values depend on the scheduler kind, so we get them
+ // as parameters.
+ global_queue_interval,
+ event_interval,
+
+ seed_generator: RngSeedGenerator::new(RngSeed::new()),
+
+ #[cfg(tokio_unstable)]
+ unhandled_panic: UnhandledPanic::Ignore,
+
+ disable_lifo_slot: false,
}
}
@@ -165,7 +289,11 @@
/// .unwrap();
/// ```
pub fn enable_all(&mut self) -> &mut Self {
- #[cfg(any(feature = "net", feature = "process", all(unix, feature = "signal")))]
+ #[cfg(any(
+ feature = "net",
+ all(unix, feature = "process"),
+ all(unix, feature = "signal")
+ ))]
self.enable_io();
#[cfg(feature = "time")]
self.enable_time();
@@ -178,15 +306,13 @@
/// This can be any number above 0 though it is advised to keep this value
/// on the smaller side.
///
+ /// This will override the value read from environment variable `TOKIO_WORKER_THREADS`.
+ ///
/// # Default
///
/// The default value is the number of cores available to the system.
///
- /// # Panic
- ///
- /// When using the `current_thread` runtime this method will panic, since
- /// those variants do not allow setting worker thread counts.
- ///
+ /// When using the `current_thread` runtime this method has no effect.
///
/// # Examples
///
@@ -219,9 +345,10 @@
/// rt.block_on(async move {});
/// ```
///
- /// # Panic
+ /// # Panics
///
/// This will panic if `val` is not larger than `0`.
+ #[track_caller]
pub fn worker_threads(&mut self, val: usize) -> &mut Self {
assert!(val > 0, "Worker threads cannot be set to 0");
self.worker_threads = Some(val);
@@ -237,7 +364,7 @@
///
/// The default value is 512.
///
- /// # Panic
+ /// # Panics
///
/// This will panic if `val` is not larger than `0`.
///
@@ -249,6 +376,7 @@
/// [`spawn_blocking`]: fn@crate::task::spawn_blocking
/// [`worker_threads`]: Self::worker_threads
/// [`thread_keep_alive`]: Self::thread_keep_alive
+ #[track_caller]
#[cfg_attr(docsrs, doc(alias = "max_threads"))]
pub fn max_blocking_threads(&mut self, val: usize) -> &mut Self {
assert!(val > 0, "Max blocking threads cannot be set to 0");
@@ -286,7 +414,6 @@
/// ```
/// # use tokio::runtime;
/// # use std::sync::atomic::{AtomicUsize, Ordering};
- ///
/// # pub fn main() {
/// let rt = runtime::Builder::new_multi_thread()
/// .thread_name_fn(|| {
@@ -338,7 +465,6 @@
///
/// ```
/// # use tokio::runtime;
- ///
/// # pub fn main() {
/// let runtime = runtime::Builder::new_multi_thread()
/// .on_thread_start(|| {
@@ -364,7 +490,6 @@
///
/// ```
/// # use tokio::runtime;
- ///
/// # pub fn main() {
/// let runtime = runtime::Builder::new_multi_thread()
/// .on_thread_stop(|| {
@@ -473,7 +598,6 @@
///
/// ```
/// # use tokio::runtime;
- ///
/// # pub fn main() {
/// let runtime = runtime::Builder::new_multi_thread()
/// .on_thread_unpark(|| {
@@ -513,8 +637,8 @@
/// ```
pub fn build(&mut self) -> io::Result<Runtime> {
match &self.kind {
- Kind::CurrentThread => self.build_basic_runtime(),
- #[cfg(feature = "rt-multi-thread")]
+ Kind::CurrentThread => self.build_current_thread_runtime(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
Kind::MultiThread => self.build_threaded_runtime(),
}
}
@@ -523,12 +647,13 @@
driver::Cfg {
enable_pause_time: match self.kind {
Kind::CurrentThread => true,
- #[cfg(feature = "rt-multi-thread")]
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
Kind::MultiThread => false,
},
enable_io: self.enable_io,
enable_time: self.enable_time,
start_paused: self.start_paused,
+ nevents: self.nevents,
}
}
@@ -542,7 +667,6 @@
/// ```
/// # use tokio::runtime;
/// # use std::time::Duration;
- ///
/// # pub fn main() {
/// let rt = runtime::Builder::new_multi_thread()
/// .thread_keep_alive(Duration::from_millis(100))
@@ -554,35 +678,249 @@
self
}
- fn build_basic_runtime(&mut self) -> io::Result<Runtime> {
- use crate::runtime::{BasicScheduler, Kind};
+ /// Sets the number of scheduler ticks after which the scheduler will poll the global
+ /// task queue.
+ ///
+ /// A scheduler "tick" roughly corresponds to one `poll` invocation on a task.
+ ///
+ /// By default the global queue interval is:
+ ///
+ /// * `31` for the current-thread scheduler.
+ /// * `61` for the multithreaded scheduler.
+ ///
+ /// Schedulers have a local queue of already-claimed tasks, and a global queue of incoming
+ /// tasks. Setting the interval to a smaller value increases the fairness of the scheduler,
+ /// at the cost of more synchronization overhead. That can be beneficial for prioritizing
+ /// getting started on new work, especially if tasks frequently yield rather than complete
+ /// or await on further I/O. Conversely, a higher value prioritizes existing work, and
+ /// is a good choice when most tasks quickly complete polling.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use tokio::runtime;
+ /// # pub fn main() {
+ /// let rt = runtime::Builder::new_multi_thread()
+ /// .global_queue_interval(31)
+ /// .build();
+ /// # }
+ /// ```
+ pub fn global_queue_interval(&mut self, val: u32) -> &mut Self {
+ self.global_queue_interval = val;
+ self
+ }
- let (driver, resources) = driver::Driver::new(self.get_cfg())?;
+ /// Sets the number of scheduler ticks after which the scheduler will poll for
+ /// external events (timers, I/O, and so on).
+ ///
+ /// A scheduler "tick" roughly corresponds to one `poll` invocation on a task.
+ ///
+ /// By default, the event interval is `61` for all scheduler types.
+ ///
+ /// Setting the event interval determines the effective "priority" of delivering
+ /// these external events (which may wake up additional tasks), compared to
+ /// executing tasks that are currently ready to run. A smaller value is useful
+ /// when tasks frequently spend a long time in polling, or frequently yield,
+ /// which can result in overly long delays picking up I/O events. Conversely,
+ /// picking up new events requires extra synchronization and syscall overhead,
+ /// so if tasks generally complete their polling quickly, a higher event interval
+ /// will minimize that overhead while still keeping the scheduler responsive to
+ /// events.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use tokio::runtime;
+ /// # pub fn main() {
+ /// let rt = runtime::Builder::new_multi_thread()
+ /// .event_interval(31)
+ /// .build();
+ /// # }
+ /// ```
+ pub fn event_interval(&mut self, val: u32) -> &mut Self {
+ self.event_interval = val;
+ self
+ }
- // And now put a single-threaded scheduler on top of the timer. When
- // there are no futures ready to do something, it'll let the timer or
- // the reactor to generate some new stimuli for the futures to continue
- // in their life.
- let scheduler =
- BasicScheduler::new(driver, self.before_park.clone(), self.after_unpark.clone());
- let spawner = Spawner::Basic(scheduler.spawner().clone());
+ cfg_unstable! {
+ /// Configure how the runtime responds to an unhandled panic on a
+ /// spawned task.
+ ///
+ /// By default, an unhandled panic (i.e. a panic not caught by
+ /// [`std::panic::catch_unwind`]) has no impact on the runtime's
+ /// execution. The panic is error value is forwarded to the task's
+ /// [`JoinHandle`] and all other spawned tasks continue running.
+ ///
+ /// The `unhandled_panic` option enables configuring this behavior.
+ ///
+ /// * `UnhandledPanic::Ignore` is the default behavior. Panics on
+ /// spawned tasks have no impact on the runtime's execution.
+ /// * `UnhandledPanic::ShutdownRuntime` will force the runtime to
+ /// shutdown immediately when a spawned task panics even if that
+ /// task's `JoinHandle` has not been dropped. All other spawned tasks
+ /// will immediately terminate and further calls to
+ /// [`Runtime::block_on`] will panic.
+ ///
+ /// # Unstable
+ ///
+ /// This option is currently unstable and its implementation is
+ /// incomplete. The API may change or be removed in the future. See
+ /// tokio-rs/tokio#4516 for more details.
+ ///
+ /// # Examples
+ ///
+ /// The following demonstrates a runtime configured to shutdown on
+ /// panic. The first spawned task panics and results in the runtime
+ /// shutting down. The second spawned task never has a chance to
+ /// execute. The call to `block_on` will panic due to the runtime being
+ /// forcibly shutdown.
+ ///
+ /// ```should_panic
+ /// use tokio::runtime::{self, UnhandledPanic};
+ ///
+ /// # pub fn main() {
+ /// let rt = runtime::Builder::new_current_thread()
+ /// .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ /// .build()
+ /// .unwrap();
+ ///
+ /// rt.spawn(async { panic!("boom"); });
+ /// rt.spawn(async {
+ /// // This task never completes.
+ /// });
+ ///
+ /// rt.block_on(async {
+ /// // Do some work
+ /// # loop { tokio::task::yield_now().await; }
+ /// })
+ /// # }
+ /// ```
+ ///
+ /// [`JoinHandle`]: struct@crate::task::JoinHandle
+ pub fn unhandled_panic(&mut self, behavior: UnhandledPanic) -> &mut Self {
+ self.unhandled_panic = behavior;
+ self
+ }
+
+ /// Disables the LIFO task scheduler heuristic.
+ ///
+ /// The multi-threaded scheduler includes a heuristic for optimizing
+ /// message-passing patterns. This heuristic results in the **last**
+ /// scheduled task being polled first.
+ ///
+ /// To implement this heuristic, each worker thread has a slot which
+ /// holds the task that should be polled next. However, this slot cannot
+ /// be stolen by other worker threads, which can result in lower total
+ /// throughput when tasks tend to have longer poll times.
+ ///
+ /// This configuration option will disable this heuristic resulting in
+ /// all scheduled tasks being pushed into the worker-local queue, which
+ /// is stealable.
+ ///
+ /// Consider trying this option when the task "scheduled" time is high
+ /// but the runtime is underutilized. Use tokio-rs/tokio-metrics to
+ /// collect this data.
+ ///
+ /// # Unstable
+ ///
+ /// This configuration option is considered a workaround for the LIFO
+ /// slot not being stealable. When the slot becomes stealable, we will
+ /// revisit whether or not this option is necessary. See
+ /// tokio-rs/tokio#4941.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime;
+ ///
+ /// let rt = runtime::Builder::new_multi_thread()
+ /// .disable_lifo_slot()
+ /// .build()
+ /// .unwrap();
+ /// ```
+ pub fn disable_lifo_slot(&mut self) -> &mut Self {
+ self.disable_lifo_slot = true;
+ self
+ }
+
+ /// Specifies the random number generation seed to use within all
+ /// threads associated with the runtime being built.
+ ///
+ /// This option is intended to make certain parts of the runtime
+ /// deterministic (e.g. the [`tokio::select!`] macro). In the case of
+ /// [`tokio::select!`] it will ensure that the order that branches are
+ /// polled is deterministic.
+ ///
+ /// In addition to the code specifying `rng_seed` and interacting with
+ /// the runtime, the internals of Tokio and the Rust compiler may affect
+ /// the sequences of random numbers. In order to ensure repeatable
+ /// results, the version of Tokio, the versions of all other
+ /// dependencies that interact with Tokio, and the Rust compiler version
+ /// should also all remain constant.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use tokio::runtime::{self, RngSeed};
+ /// # pub fn main() {
+ /// let seed = RngSeed::from_bytes(b"place your seed here");
+ /// let rt = runtime::Builder::new_current_thread()
+ /// .rng_seed(seed)
+ /// .build();
+ /// # }
+ /// ```
+ ///
+ /// [`tokio::select!`]: crate::select
+ pub fn rng_seed(&mut self, seed: RngSeed) -> &mut Self {
+ self.seed_generator = RngSeedGenerator::new(seed);
+ self
+ }
+ }
+
+ fn build_current_thread_runtime(&mut self) -> io::Result<Runtime> {
+ use crate::runtime::scheduler::{self, CurrentThread};
+ use crate::runtime::{runtime::Scheduler, Config};
+
+ let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?;
// Blocking pool
let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads);
let blocking_spawner = blocking_pool.spawner().clone();
- Ok(Runtime {
- kind: Kind::CurrentThread(scheduler),
- handle: Handle {
- spawner,
- io_handle: resources.io_handle,
- time_handle: resources.time_handle,
- signal_handle: resources.signal_handle,
- clock: resources.clock,
- blocking_spawner,
+ // Generate a rng seed for this runtime.
+ let seed_generator_1 = self.seed_generator.next_generator();
+ let seed_generator_2 = self.seed_generator.next_generator();
+
+ // And now put a single-threaded scheduler on top of the timer. When
+ // there are no futures ready to do something, it'll let the timer or
+ // the reactor to generate some new stimuli for the futures to continue
+ // in their life.
+ let (scheduler, handle) = CurrentThread::new(
+ driver,
+ driver_handle,
+ blocking_spawner,
+ seed_generator_2,
+ Config {
+ before_park: self.before_park.clone(),
+ after_unpark: self.after_unpark.clone(),
+ global_queue_interval: self.global_queue_interval,
+ event_interval: self.event_interval,
+ #[cfg(tokio_unstable)]
+ unhandled_panic: self.unhandled_panic.clone(),
+ disable_lifo_slot: self.disable_lifo_slot,
+ seed_generator: seed_generator_1,
},
+ );
+
+ let handle = Handle {
+ inner: scheduler::Handle::CurrentThread(handle),
+ };
+
+ Ok(Runtime::from_parts(
+ Scheduler::CurrentThread(scheduler),
+ handle,
blocking_pool,
- })
+ ))
}
}
@@ -607,6 +945,25 @@
self.enable_io = true;
self
}
+
+ /// Enables the I/O driver and configures the max number of events to be
+ /// processed per tick.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime;
+ ///
+ /// let rt = runtime::Builder::new_current_thread()
+ /// .enable_io()
+ /// .max_io_events_per_tick(1024)
+ /// .build()
+ /// .unwrap();
+ /// ```
+ pub fn max_io_events_per_tick(&mut self, capacity: usize) -> &mut Self {
+ self.nevents = capacity;
+ self
+ }
}
}
@@ -662,39 +1019,47 @@
impl Builder {
fn build_threaded_runtime(&mut self) -> io::Result<Runtime> {
use crate::loom::sys::num_cpus;
- use crate::runtime::{Kind, ThreadPool};
- use crate::runtime::park::Parker;
+ use crate::runtime::{Config, runtime::Scheduler};
+ use crate::runtime::scheduler::{self, MultiThread};
let core_threads = self.worker_threads.unwrap_or_else(num_cpus);
- let (driver, resources) = driver::Driver::new(self.get_cfg())?;
-
- let (scheduler, launch) = ThreadPool::new(core_threads, Parker::new(driver), self.before_park.clone(), self.after_unpark.clone());
- let spawner = Spawner::ThreadPool(scheduler.spawner().clone());
+ let (driver, driver_handle) = driver::Driver::new(self.get_cfg())?;
// Create the blocking pool
- let blocking_pool = blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads);
+ let blocking_pool =
+ blocking::create_blocking_pool(self, self.max_blocking_threads + core_threads);
let blocking_spawner = blocking_pool.spawner().clone();
- // Create the runtime handle
- let handle = Handle {
- spawner,
- io_handle: resources.io_handle,
- time_handle: resources.time_handle,
- signal_handle: resources.signal_handle,
- clock: resources.clock,
+ // Generate a rng seed for this runtime.
+ let seed_generator_1 = self.seed_generator.next_generator();
+ let seed_generator_2 = self.seed_generator.next_generator();
+
+ let (scheduler, handle, launch) = MultiThread::new(
+ core_threads,
+ driver,
+ driver_handle,
blocking_spawner,
- };
+ seed_generator_2,
+ Config {
+ before_park: self.before_park.clone(),
+ after_unpark: self.after_unpark.clone(),
+ global_queue_interval: self.global_queue_interval,
+ event_interval: self.event_interval,
+ #[cfg(tokio_unstable)]
+ unhandled_panic: self.unhandled_panic.clone(),
+ disable_lifo_slot: self.disable_lifo_slot,
+ seed_generator: seed_generator_1,
+ },
+ );
+
+ let handle = Handle { inner: scheduler::Handle::MultiThread(handle) };
// Spawn the thread pool workers
- let _enter = crate::runtime::context::enter(handle.clone());
+ let _enter = handle.enter();
launch.launch();
- Ok(Runtime {
- kind: Kind::ThreadPool(scheduler),
- handle,
- blocking_pool,
- })
+ Ok(Runtime::from_parts(Scheduler::MultiThread(scheduler), handle, blocking_pool))
}
}
}
diff --git a/src/runtime/config.rs b/src/runtime/config.rs
new file mode 100644
index 0000000..39eb1cf
--- /dev/null
+++ b/src/runtime/config.rs
@@ -0,0 +1,34 @@
+#![cfg_attr(any(not(feature = "full"), tokio_wasm), allow(dead_code))]
+use crate::runtime::Callback;
+use crate::util::RngSeedGenerator;
+
+pub(crate) struct Config {
+ /// How many ticks before pulling a task from the global/remote queue?
+ pub(crate) global_queue_interval: u32,
+
+ /// How many ticks before yielding to the driver for timer and I/O events?
+ pub(crate) event_interval: u32,
+
+ /// Callback for a worker parking itself
+ pub(crate) before_park: Option<Callback>,
+
+ /// Callback for a worker unparking itself
+ pub(crate) after_unpark: Option<Callback>,
+
+ /// The multi-threaded scheduler includes a per-worker LIFO slot used to
+ /// store the last scheduled task. This can improve certain usage patterns,
+ /// especially message passing between tasks. However, this LIFO slot is not
+ /// currently stealable.
+ ///
+ /// Eventually, the LIFO slot **will** become stealable, however as a
+ /// stop-gap, this unstable option lets users disable the LIFO task.
+ pub(crate) disable_lifo_slot: bool,
+
+ /// Random number generator seed to configure runtimes to act in a
+ /// deterministic way.
+ pub(crate) seed_generator: RngSeedGenerator,
+
+ #[cfg(tokio_unstable)]
+ /// How to respond to unhandled task panics.
+ pub(crate) unhandled_panic: crate::runtime::UnhandledPanic,
+}
diff --git a/src/runtime/context.rs b/src/runtime/context.rs
index 1f44a53..fef53ca 100644
--- a/src/runtime/context.rs
+++ b/src/runtime/context.rs
@@ -1,111 +1,420 @@
-//! Thread local runtime context
-use crate::runtime::{Handle, TryCurrentError};
+use crate::loom::thread::AccessError;
+use crate::runtime::coop;
-use std::cell::RefCell;
+use std::cell::Cell;
-thread_local! {
- static CONTEXT: RefCell<Option<Handle>> = RefCell::new(None)
+#[cfg(any(feature = "rt", feature = "macros"))]
+use crate::util::rand::{FastRand, RngSeed};
+
+cfg_rt! {
+ use crate::runtime::{scheduler, task::Id, Defer};
+
+ use std::cell::RefCell;
+ use std::marker::PhantomData;
+ use std::time::Duration;
}
-pub(crate) fn try_current() -> Result<Handle, crate::runtime::TryCurrentError> {
- match CONTEXT.try_with(|ctx| ctx.borrow().clone()) {
- Ok(Some(handle)) => Ok(handle),
- Ok(None) => Err(TryCurrentError::new_no_context()),
- Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()),
- }
+struct Context {
+ /// Uniquely identifies the current thread
+ #[cfg(feature = "rt")]
+ thread_id: Cell<Option<ThreadId>>,
+
+ /// Handle to the runtime scheduler running on the current thread.
+ #[cfg(feature = "rt")]
+ handle: RefCell<Option<scheduler::Handle>>,
+
+ #[cfg(feature = "rt")]
+ current_task_id: Cell<Option<Id>>,
+
+ /// Tracks if the current thread is currently driving a runtime.
+ /// Note, that if this is set to "entered", the current scheduler
+ /// handle may not reference the runtime currently executing. This
+ /// is because other runtime handles may be set to current from
+ /// within a runtime.
+ #[cfg(feature = "rt")]
+ runtime: Cell<EnterRuntime>,
+
+ /// Yielded task wakers are stored here and notified after resource drivers
+ /// are polled.
+ #[cfg(feature = "rt")]
+ defer: RefCell<Option<Defer>>,
+
+ #[cfg(any(feature = "rt", feature = "macros"))]
+ rng: FastRand,
+
+ /// Tracks the amount of "work" a task may still do before yielding back to
+ /// the sheduler
+ budget: Cell<coop::Budget>,
}
-pub(crate) fn current() -> Handle {
- match try_current() {
- Ok(handle) => handle,
- Err(e) => panic!("{}", e),
- }
-}
+tokio_thread_local! {
+ static CONTEXT: Context = {
+ Context {
+ #[cfg(feature = "rt")]
+ thread_id: Cell::new(None),
-cfg_io_driver! {
- pub(crate) fn io_handle() -> crate::runtime::driver::IoHandle {
- match CONTEXT.try_with(|ctx| {
- let ctx = ctx.borrow();
- ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).io_handle.clone()
- }) {
- Ok(io_handle) => io_handle,
- Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
+ /// Tracks the current runtime handle to use when spawning,
+ /// accessing drivers, etc...
+ #[cfg(feature = "rt")]
+ handle: RefCell::new(None),
+ #[cfg(feature = "rt")]
+ current_task_id: Cell::new(None),
+
+ /// Tracks if the current thread is currently driving a runtime.
+ /// Note, that if this is set to "entered", the current scheduler
+ /// handle may not reference the runtime currently executing. This
+ /// is because other runtime handles may be set to current from
+ /// within a runtime.
+ #[cfg(feature = "rt")]
+ runtime: Cell::new(EnterRuntime::NotEntered),
+
+ #[cfg(feature = "rt")]
+ defer: RefCell::new(None),
+
+ #[cfg(any(feature = "rt", feature = "macros"))]
+ rng: FastRand::new(RngSeed::new()),
+
+ budget: Cell::new(coop::Budget::unconstrained()),
}
}
}
-cfg_signal_internal! {
- #[cfg(unix)]
- pub(crate) fn signal_handle() -> crate::runtime::driver::SignalHandle {
- match CONTEXT.try_with(|ctx| {
- let ctx = ctx.borrow();
- ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).signal_handle.clone()
- }) {
- Ok(signal_handle) => signal_handle,
- Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
- }
- }
+#[cfg(feature = "macros")]
+pub(crate) fn thread_rng_n(n: u32) -> u32 {
+ CONTEXT.with(|ctx| ctx.rng.fastrand_n(n))
}
-cfg_time! {
- pub(crate) fn time_handle() -> crate::runtime::driver::TimeHandle {
- match CONTEXT.try_with(|ctx| {
- let ctx = ctx.borrow();
- ctx.as_ref().expect(crate::util::error::CONTEXT_MISSING_ERROR).time_handle.clone()
- }) {
- Ok(time_handle) => time_handle,
- Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
- }
- }
-
- cfg_test_util! {
- pub(crate) fn clock() -> Option<crate::runtime::driver::Clock> {
- match CONTEXT.try_with(|ctx| (*ctx.borrow()).as_ref().map(|ctx| ctx.clock.clone())) {
- Ok(clock) => clock,
- Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
- }
- }
- }
+pub(super) fn budget<R>(f: impl FnOnce(&Cell<coop::Budget>) -> R) -> Result<R, AccessError> {
+ CONTEXT.try_with(|ctx| f(&ctx.budget))
}
cfg_rt! {
- pub(crate) fn spawn_handle() -> Option<crate::runtime::Spawner> {
- match CONTEXT.try_with(|ctx| (*ctx.borrow()).as_ref().map(|ctx| ctx.spawner.clone())) {
- Ok(spawner) => spawner,
- Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
+ use crate::runtime::{ThreadId, TryCurrentError};
+
+ use std::fmt;
+
+ pub(crate) fn thread_id() -> Result<ThreadId, AccessError> {
+ CONTEXT.try_with(|ctx| {
+ match ctx.thread_id.get() {
+ Some(id) => id,
+ None => {
+ let id = ThreadId::next();
+ ctx.thread_id.set(Some(id));
+ id
+ }
+ }
+ })
+ }
+
+ #[derive(Debug, Clone, Copy)]
+ #[must_use]
+ pub(crate) enum EnterRuntime {
+ /// Currently in a runtime context.
+ #[cfg_attr(not(feature = "rt"), allow(dead_code))]
+ Entered { allow_block_in_place: bool },
+
+ /// Not in a runtime context **or** a blocking region.
+ NotEntered,
+ }
+
+ #[derive(Debug)]
+ #[must_use]
+ pub(crate) struct SetCurrentGuard {
+ old_handle: Option<scheduler::Handle>,
+ old_seed: RngSeed,
+ }
+
+ /// Guard tracking that a caller has entered a runtime context.
+ #[must_use]
+ pub(crate) struct EnterRuntimeGuard {
+ /// Tracks that the current thread has entered a blocking function call.
+ pub(crate) blocking: BlockingRegionGuard,
+
+ #[allow(dead_code)] // Only tracking the guard.
+ pub(crate) handle: SetCurrentGuard,
+
+ /// If true, then this is the root runtime guard. It is possible to nest
+ /// runtime guards by using `block_in_place` between the calls. We need
+ /// to track the root guard as this is the guard responsible for freeing
+ /// the deferred task queue.
+ is_root: bool,
+ }
+
+ /// Guard tracking that a caller has entered a blocking region.
+ #[must_use]
+ pub(crate) struct BlockingRegionGuard {
+ _p: PhantomData<RefCell<()>>,
+ }
+
+ pub(crate) struct DisallowBlockInPlaceGuard(bool);
+
+ pub(crate) fn set_current_task_id(id: Option<Id>) -> Option<Id> {
+ CONTEXT.try_with(|ctx| ctx.current_task_id.replace(id)).unwrap_or(None)
+ }
+
+ pub(crate) fn current_task_id() -> Option<Id> {
+ CONTEXT.try_with(|ctx| ctx.current_task_id.get()).unwrap_or(None)
+ }
+
+ pub(crate) fn try_current() -> Result<scheduler::Handle, TryCurrentError> {
+ match CONTEXT.try_with(|ctx| ctx.handle.borrow().clone()) {
+ Ok(Some(handle)) => Ok(handle),
+ Ok(None) => Err(TryCurrentError::new_no_context()),
+ Err(_access_error) => Err(TryCurrentError::new_thread_local_destroyed()),
+ }
+ }
+
+ /// Sets this [`Handle`] as the current active [`Handle`].
+ ///
+ /// [`Handle`]: crate::runtime::scheduler::Handle
+ pub(crate) fn try_set_current(handle: &scheduler::Handle) -> Option<SetCurrentGuard> {
+ CONTEXT.try_with(|ctx| ctx.set_current(handle)).ok()
+ }
+
+
+ /// Marks the current thread as being within the dynamic extent of an
+ /// executor.
+ #[track_caller]
+ pub(crate) fn enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool) -> EnterRuntimeGuard {
+ if let Some(enter) = try_enter_runtime(handle, allow_block_in_place) {
+ return enter;
+ }
+
+ panic!(
+ "Cannot start a runtime from within a runtime. This happens \
+ because a function (like `block_on`) attempted to block the \
+ current thread while the thread is being used to drive \
+ asynchronous tasks."
+ );
+ }
+
+ /// Tries to enter a runtime context, returns `None` if already in a runtime
+ /// context.
+ fn try_enter_runtime(handle: &scheduler::Handle, allow_block_in_place: bool) -> Option<EnterRuntimeGuard> {
+ CONTEXT.with(|c| {
+ if c.runtime.get().is_entered() {
+ None
+ } else {
+ // Set the entered flag
+ c.runtime.set(EnterRuntime::Entered { allow_block_in_place });
+
+ // Initialize queue to track yielded tasks
+ let mut defer = c.defer.borrow_mut();
+
+ let is_root = if defer.is_none() {
+ *defer = Some(Defer::new());
+ true
+ } else {
+ false
+ };
+
+ Some(EnterRuntimeGuard {
+ blocking: BlockingRegionGuard::new(),
+ handle: c.set_current(handle),
+ is_root,
+ })
+ }
+ })
+ }
+
+ pub(crate) fn try_enter_blocking_region() -> Option<BlockingRegionGuard> {
+ CONTEXT.try_with(|c| {
+ if c.runtime.get().is_entered() {
+ None
+ } else {
+ Some(BlockingRegionGuard::new())
+ }
+ // If accessing the thread-local fails, the thread is terminating
+ // and thread-locals are being destroyed. Because we don't know if
+ // we are currently in a runtime or not, we default to being
+ // permissive.
+ }).unwrap_or_else(|_| Some(BlockingRegionGuard::new()))
+ }
+
+ /// Disallows blocking in the current runtime context until the guard is dropped.
+ pub(crate) fn disallow_block_in_place() -> DisallowBlockInPlaceGuard {
+ let reset = CONTEXT.with(|c| {
+ if let EnterRuntime::Entered {
+ allow_block_in_place: true,
+ } = c.runtime.get()
+ {
+ c.runtime.set(EnterRuntime::Entered {
+ allow_block_in_place: false,
+ });
+ true
+ } else {
+ false
+ }
+ });
+
+ DisallowBlockInPlaceGuard(reset)
+ }
+
+ pub(crate) fn with_defer<R>(f: impl FnOnce(&mut Defer) -> R) -> Option<R> {
+ CONTEXT.with(|c| {
+ let mut defer = c.defer.borrow_mut();
+ defer.as_mut().map(f)
+ })
+ }
+
+ impl Context {
+ fn set_current(&self, handle: &scheduler::Handle) -> SetCurrentGuard {
+ let rng_seed = handle.seed_generator().next_seed();
+
+ let old_handle = self.handle.borrow_mut().replace(handle.clone());
+ let old_seed = self.rng.replace_seed(rng_seed);
+
+ SetCurrentGuard {
+ old_handle,
+ old_seed,
+ }
+ }
+ }
+
+ impl Drop for SetCurrentGuard {
+ fn drop(&mut self) {
+ CONTEXT.with(|ctx| {
+ *ctx.handle.borrow_mut() = self.old_handle.take();
+ ctx.rng.replace_seed(self.old_seed.clone());
+ });
+ }
+ }
+
+ impl fmt::Debug for EnterRuntimeGuard {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("Enter").finish()
+ }
+ }
+
+ impl Drop for EnterRuntimeGuard {
+ fn drop(&mut self) {
+ CONTEXT.with(|c| {
+ assert!(c.runtime.get().is_entered());
+ c.runtime.set(EnterRuntime::NotEntered);
+
+ if self.is_root {
+ *c.defer.borrow_mut() = None;
+ }
+ });
+ }
+ }
+
+ impl BlockingRegionGuard {
+ fn new() -> BlockingRegionGuard {
+ BlockingRegionGuard { _p: PhantomData }
+ }
+ /// Blocks the thread on the specified future, returning the value with
+ /// which that future completes.
+ pub(crate) fn block_on<F>(&mut self, f: F) -> Result<F::Output, AccessError>
+ where
+ F: std::future::Future,
+ {
+ use crate::runtime::park::CachedParkThread;
+
+ let mut park = CachedParkThread::new();
+ park.block_on(f)
+ }
+
+ /// Blocks the thread on the specified future for **at most** `timeout`
+ ///
+ /// If the future completes before `timeout`, the result is returned. If
+ /// `timeout` elapses, then `Err` is returned.
+ pub(crate) fn block_on_timeout<F>(&mut self, f: F, timeout: Duration) -> Result<F::Output, ()>
+ where
+ F: std::future::Future,
+ {
+ use crate::runtime::park::CachedParkThread;
+ use std::task::Context;
+ use std::task::Poll::Ready;
+ use std::time::Instant;
+
+ let mut park = CachedParkThread::new();
+ let waker = park.waker().map_err(|_| ())?;
+ let mut cx = Context::from_waker(&waker);
+
+ pin!(f);
+ let when = Instant::now() + timeout;
+
+ loop {
+ if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) {
+ return Ok(v);
+ }
+
+ let now = Instant::now();
+
+ if now >= when {
+ return Err(());
+ }
+
+ // Wake any yielded tasks before parking in order to avoid
+ // blocking.
+ with_defer(|defer| defer.wake());
+
+ park.park_timeout(when - now);
+ }
+ }
+ }
+
+ impl Drop for DisallowBlockInPlaceGuard {
+ fn drop(&mut self) {
+ if self.0 {
+ // XXX: Do we want some kind of assertion here, or is "best effort" okay?
+ CONTEXT.with(|c| {
+ if let EnterRuntime::Entered {
+ allow_block_in_place: false,
+ } = c.runtime.get()
+ {
+ c.runtime.set(EnterRuntime::Entered {
+ allow_block_in_place: true,
+ });
+ }
+ })
+ }
+ }
+ }
+
+ impl EnterRuntime {
+ pub(crate) fn is_entered(self) -> bool {
+ matches!(self, EnterRuntime::Entered { .. })
}
}
}
-/// Sets this [`Handle`] as the current active [`Handle`].
-///
-/// [`Handle`]: Handle
-pub(crate) fn enter(new: Handle) -> EnterGuard {
- match try_enter(new) {
- Some(guard) => guard,
- None => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
+// Forces the current "entered" state to be cleared while the closure
+// is executed.
+//
+// # Warning
+//
+// This is hidden for a reason. Do not use without fully understanding
+// executors. Misusing can easily cause your program to deadlock.
+cfg_rt_multi_thread! {
+ /// Returns true if in a runtime context.
+ pub(crate) fn current_enter_context() -> EnterRuntime {
+ CONTEXT.with(|c| c.runtime.get())
}
-}
-/// Sets this [`Handle`] as the current active [`Handle`].
-///
-/// [`Handle`]: Handle
-pub(crate) fn try_enter(new: Handle) -> Option<EnterGuard> {
- CONTEXT
- .try_with(|ctx| {
- let old = ctx.borrow_mut().replace(new);
- EnterGuard(old)
- })
- .ok()
-}
+ pub(crate) fn exit_runtime<F: FnOnce() -> R, R>(f: F) -> R {
+ // Reset in case the closure panics
+ struct Reset(EnterRuntime);
-#[derive(Debug)]
-pub(crate) struct EnterGuard(Option<Handle>);
+ impl Drop for Reset {
+ fn drop(&mut self) {
+ CONTEXT.with(|c| {
+ assert!(!c.runtime.get().is_entered(), "closure claimed permanent executor");
+ c.runtime.set(self.0);
+ });
+ }
+ }
-impl Drop for EnterGuard {
- fn drop(&mut self) {
- CONTEXT.with(|ctx| {
- *ctx.borrow_mut() = self.0.take();
+ let was = CONTEXT.with(|c| {
+ let e = c.runtime.get();
+ assert!(e.is_entered(), "asked to exit when not entered");
+ c.runtime.set(EnterRuntime::NotEntered);
+ e
});
+
+ let _reset = Reset(was);
+ // dropping _reset after f() will reset ENTERED
+ f()
}
}
diff --git a/src/coop.rs b/src/runtime/coop.rs
similarity index 84%
rename from src/coop.rs
rename to src/runtime/coop.rs
index 256e962..0ba137a 100644
--- a/src/coop.rs
+++ b/src/runtime/coop.rs
@@ -29,11 +29,7 @@
// other futures. By doing this, you avoid double-counting each iteration of
// the outer future against the cooperating budget.
-use std::cell::Cell;
-
-thread_local! {
- static CURRENT: Cell<Budget> = Cell::new(Budget::unconstrained());
-}
+use crate::runtime::context;
/// Opaque type tracking the amount of "work" a task may still do before
/// yielding back to the scheduler.
@@ -56,16 +52,12 @@
}
/// Returns an unconstrained budget. Operations will not be limited.
- const fn unconstrained() -> Budget {
+ pub(super) const fn unconstrained() -> Budget {
Budget(None)
}
-}
-cfg_rt_multi_thread! {
- impl Budget {
- fn has_remaining(self) -> bool {
- self.0.map(|budget| budget > 0).unwrap_or(true)
- }
+ fn has_remaining(self) -> bool {
+ self.0.map(|budget| budget > 0).unwrap_or(true)
}
}
@@ -85,37 +77,42 @@
#[inline(always)]
fn with_budget<R>(budget: Budget, f: impl FnOnce() -> R) -> R {
- struct ResetGuard<'a> {
- cell: &'a Cell<Budget>,
+ struct ResetGuard {
prev: Budget,
}
- impl<'a> Drop for ResetGuard<'a> {
+ impl Drop for ResetGuard {
fn drop(&mut self) {
- self.cell.set(self.prev);
+ let _ = context::budget(|cell| {
+ cell.set(self.prev);
+ });
}
}
- CURRENT.with(move |cell| {
+ #[allow(unused_variables)]
+ let maybe_guard = context::budget(|cell| {
let prev = cell.get();
-
cell.set(budget);
- let _guard = ResetGuard { cell, prev };
+ ResetGuard { prev }
+ });
- f()
- })
+ // The function is called regardless even if the budget is not successfully
+ // set due to the thread-local being destroyed.
+ f()
+}
+
+#[inline(always)]
+pub(crate) fn has_budget_remaining() -> bool {
+ // If the current budget cannot be accessed due to the thread-local being
+ // shutdown, then we assume there is budget remaining.
+ context::budget(|cell| cell.get().has_remaining()).unwrap_or(true)
}
cfg_rt_multi_thread! {
/// Sets the current task's budget.
pub(crate) fn set(budget: Budget) {
- CURRENT.with(|cell| cell.set(budget))
- }
-
- #[inline(always)]
- pub(crate) fn has_budget_remaining() -> bool {
- CURRENT.with(|cell| cell.get().has_remaining())
+ let _ = context::budget(|cell| cell.set(budget));
}
}
@@ -124,15 +121,16 @@
///
/// Returns the remaining budget
pub(crate) fn stop() -> Budget {
- CURRENT.with(|cell| {
+ context::budget(|cell| {
let prev = cell.get();
cell.set(Budget::unconstrained());
prev
- })
+ }).unwrap_or(Budget::unconstrained())
}
}
cfg_coop! {
+ use std::cell::Cell;
use std::task::{Context, Poll};
#[must_use]
@@ -150,7 +148,7 @@
// They are both represented as the remembered budget being unconstrained.
let budget = self.0.get();
if !budget.is_unconstrained() {
- CURRENT.with(|cell| {
+ let _ = context::budget(|cell| {
cell.set(budget);
});
}
@@ -166,12 +164,12 @@
/// that the budget empties appropriately.
///
/// Note that `RestoreOnPending` restores the budget **as it was before `poll_proceed`**.
- /// Therefore, if the budget is _further_ adjusted between when `poll_proceed` returns and
+ /// Therefore, if the budget is _fCURRENT.withurther_ adjusted between when `poll_proceed` returns and
/// `RestRestoreOnPending` is dropped, those adjustments are erased unless the caller indicates
/// that progress was made.
#[inline]
pub(crate) fn poll_proceed(cx: &mut Context<'_>) -> Poll<RestoreOnPending> {
- CURRENT.with(|cell| {
+ context::budget(|cell| {
let mut budget = cell.get();
if budget.decrement() {
@@ -182,7 +180,7 @@
cx.waker().wake_by_ref();
Poll::Pending
}
- })
+ }).unwrap_or(Poll::Ready(RestoreOnPending(Cell::new(Budget::unconstrained()))))
}
impl Budget {
@@ -211,12 +209,15 @@
mod test {
use super::*;
+ #[cfg(tokio_wasm_not_wasi)]
+ use wasm_bindgen_test::wasm_bindgen_test as test;
+
fn get() -> Budget {
- CURRENT.with(|cell| cell.get())
+ context::budget(|cell| cell.get()).unwrap_or(Budget::unconstrained())
}
#[test]
- fn bugeting() {
+ fn budgeting() {
use futures::future::poll_fn;
use tokio_test::*;
diff --git a/src/runtime/defer.rs b/src/runtime/defer.rs
new file mode 100644
index 0000000..4078512
--- /dev/null
+++ b/src/runtime/defer.rs
@@ -0,0 +1,27 @@
+use std::task::Waker;
+
+pub(crate) struct Defer {
+ deferred: Vec<Waker>,
+}
+
+impl Defer {
+ pub(crate) fn new() -> Defer {
+ Defer {
+ deferred: Default::default(),
+ }
+ }
+
+ pub(crate) fn defer(&mut self, waker: Waker) {
+ self.deferred.push(waker);
+ }
+
+ pub(crate) fn is_empty(&self) -> bool {
+ self.deferred.is_empty()
+ }
+
+ pub(crate) fn wake(&mut self) {
+ for waker in self.deferred.drain(..) {
+ waker.wake();
+ }
+ }
+}
diff --git a/src/runtime/driver.rs b/src/runtime/driver.rs
index 7e45977..4fb6b87 100644
--- a/src/runtime/driver.rs
+++ b/src/runtime/driver.rs
@@ -1,63 +1,236 @@
//! Abstracts out the entire chain of runtime sub-drivers into common types.
-use crate::park::thread::ParkThread;
-use crate::park::Park;
+
+// Eventually, this file will see significant refactoring / cleanup. For now, we
+// don't need to worry much about dead code with certain feature permutations.
+#![cfg_attr(not(feature = "full"), allow(dead_code))]
+
+use crate::runtime::park::{ParkThread, UnparkThread};
use std::io;
use std::time::Duration;
+#[derive(Debug)]
+pub(crate) struct Driver {
+ inner: TimeDriver,
+}
+
+#[derive(Debug)]
+pub(crate) struct Handle {
+ /// IO driver handle
+ pub(crate) io: IoHandle,
+
+ /// Signal driver handle
+ #[cfg_attr(any(not(unix), loom), allow(dead_code))]
+ pub(crate) signal: SignalHandle,
+
+ /// Time driver handle
+ pub(crate) time: TimeHandle,
+
+ /// Source of `Instant::now()`
+ #[cfg_attr(not(all(feature = "time", feature = "test-util")), allow(dead_code))]
+ pub(crate) clock: Clock,
+}
+
+pub(crate) struct Cfg {
+ pub(crate) enable_io: bool,
+ pub(crate) enable_time: bool,
+ pub(crate) enable_pause_time: bool,
+ pub(crate) start_paused: bool,
+ pub(crate) nevents: usize,
+}
+
+impl Driver {
+ pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Handle)> {
+ let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io, cfg.nevents)?;
+
+ let clock = create_clock(cfg.enable_pause_time, cfg.start_paused);
+
+ let (time_driver, time_handle) =
+ create_time_driver(cfg.enable_time, io_stack, clock.clone());
+
+ Ok((
+ Self { inner: time_driver },
+ Handle {
+ io: io_handle,
+ signal: signal_handle,
+ time: time_handle,
+ clock,
+ },
+ ))
+ }
+
+ pub(crate) fn park(&mut self, handle: &Handle) {
+ self.inner.park(handle)
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) {
+ self.inner.park_timeout(handle, duration)
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &Handle) {
+ self.inner.shutdown(handle)
+ }
+}
+
+impl Handle {
+ pub(crate) fn unpark(&self) {
+ #[cfg(feature = "time")]
+ if let Some(handle) = &self.time {
+ handle.unpark();
+ }
+
+ self.io.unpark();
+ }
+
+ cfg_io_driver! {
+ #[track_caller]
+ pub(crate) fn io(&self) -> &crate::runtime::io::Handle {
+ self.io
+ .as_ref()
+ .expect("A Tokio 1.x context was found, but IO is disabled. Call `enable_io` on the runtime builder to enable IO.")
+ }
+ }
+
+ cfg_signal_internal_and_unix! {
+ #[track_caller]
+ pub(crate) fn signal(&self) -> &crate::runtime::signal::Handle {
+ self.signal
+ .as_ref()
+ .expect("there is no signal driver running, must be called from the context of Tokio runtime")
+ }
+ }
+
+ cfg_time! {
+ /// Returns a reference to the time driver handle.
+ ///
+ /// Panics if no time driver is present.
+ #[track_caller]
+ pub(crate) fn time(&self) -> &crate::runtime::time::Handle {
+ self.time
+ .as_ref()
+ .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.")
+ }
+
+ cfg_test_util! {
+ pub(crate) fn clock(&self) -> &Clock {
+ &self.clock
+ }
+ }
+ }
+}
+
// ===== io driver =====
cfg_io_driver! {
- type IoDriver = crate::io::driver::Driver;
- type IoStack = crate::park::either::Either<ProcessDriver, ParkThread>;
- pub(crate) type IoHandle = Option<crate::io::driver::Handle>;
+ pub(crate) type IoDriver = crate::runtime::io::Driver;
- fn create_io_stack(enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
- use crate::park::either::Either;
+ #[derive(Debug)]
+ pub(crate) enum IoStack {
+ Enabled(ProcessDriver),
+ Disabled(ParkThread),
+ }
+ #[derive(Debug)]
+ pub(crate) enum IoHandle {
+ Enabled(crate::runtime::io::Handle),
+ Disabled(UnparkThread),
+ }
+
+ fn create_io_stack(enabled: bool, nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
#[cfg(loom)]
assert!(!enabled);
let ret = if enabled {
- let io_driver = crate::io::driver::Driver::new()?;
- let io_handle = io_driver.handle();
+ let (io_driver, io_handle) = crate::runtime::io::Driver::new(nevents)?;
- let (signal_driver, signal_handle) = create_signal_driver(io_driver)?;
+ let (signal_driver, signal_handle) = create_signal_driver(io_driver, &io_handle)?;
let process_driver = create_process_driver(signal_driver);
- (Either::A(process_driver), Some(io_handle), signal_handle)
+ (IoStack::Enabled(process_driver), IoHandle::Enabled(io_handle), signal_handle)
} else {
- (Either::B(ParkThread::new()), Default::default(), Default::default())
+ let park_thread = ParkThread::new();
+ let unpark_thread = park_thread.unpark();
+ (IoStack::Disabled(park_thread), IoHandle::Disabled(unpark_thread), Default::default())
};
Ok(ret)
}
+
+ impl IoStack {
+ pub(crate) fn park(&mut self, handle: &Handle) {
+ match self {
+ IoStack::Enabled(v) => v.park(handle),
+ IoStack::Disabled(v) => v.park(),
+ }
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) {
+ match self {
+ IoStack::Enabled(v) => v.park_timeout(handle, duration),
+ IoStack::Disabled(v) => v.park_timeout(duration),
+ }
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &Handle) {
+ match self {
+ IoStack::Enabled(v) => v.shutdown(handle),
+ IoStack::Disabled(v) => v.shutdown(),
+ }
+ }
+ }
+
+ impl IoHandle {
+ pub(crate) fn unpark(&self) {
+ match self {
+ IoHandle::Enabled(handle) => handle.unpark(),
+ IoHandle::Disabled(handle) => handle.unpark(),
+ }
+ }
+
+ pub(crate) fn as_ref(&self) -> Option<&crate::runtime::io::Handle> {
+ match self {
+ IoHandle::Enabled(v) => Some(v),
+ IoHandle::Disabled(..) => None,
+ }
+ }
+ }
}
cfg_not_io_driver! {
- pub(crate) type IoHandle = ();
- type IoStack = ParkThread;
+ pub(crate) type IoHandle = UnparkThread;
- fn create_io_stack(_enabled: bool) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
- Ok((ParkThread::new(), Default::default(), Default::default()))
+ #[derive(Debug)]
+ pub(crate) struct IoStack(ParkThread);
+
+ fn create_io_stack(_enabled: bool, _nevents: usize) -> io::Result<(IoStack, IoHandle, SignalHandle)> {
+ let park_thread = ParkThread::new();
+ let unpark_thread = park_thread.unpark();
+ Ok((IoStack(park_thread), unpark_thread, Default::default()))
+ }
+
+ impl IoStack {
+ pub(crate) fn park(&mut self, _handle: &Handle) {
+ self.0.park();
+ }
+
+ pub(crate) fn park_timeout(&mut self, _handle: &Handle, duration: Duration) {
+ self.0.park_timeout(duration);
+ }
+
+ pub(crate) fn shutdown(&mut self, _handle: &Handle) {
+ self.0.shutdown();
+ }
}
}
// ===== signal driver =====
-macro_rules! cfg_signal_internal_and_unix {
- ($($item:item)*) => {
- #[cfg(unix)]
- cfg_signal_internal! { $($item)* }
- }
-}
-
cfg_signal_internal_and_unix! {
- type SignalDriver = crate::signal::unix::driver::Driver;
- pub(crate) type SignalHandle = Option<crate::signal::unix::driver::Handle>;
+ type SignalDriver = crate::runtime::signal::Driver;
+ pub(crate) type SignalHandle = Option<crate::runtime::signal::Handle>;
- fn create_signal_driver(io_driver: IoDriver) -> io::Result<(SignalDriver, SignalHandle)> {
- let driver = crate::signal::unix::driver::Driver::new(io_driver)?;
+ fn create_signal_driver(io_driver: IoDriver, io_handle: &crate::runtime::io::Handle) -> io::Result<(SignalDriver, SignalHandle)> {
+ let driver = crate::runtime::signal::Driver::new(io_driver, io_handle)?;
let handle = driver.handle();
Ok((driver, Some(handle)))
}
@@ -69,7 +242,7 @@
cfg_io_driver! {
type SignalDriver = IoDriver;
- fn create_signal_driver(io_driver: IoDriver) -> io::Result<(SignalDriver, SignalHandle)> {
+ fn create_signal_driver(io_driver: IoDriver, _io_handle: &crate::runtime::io::Handle) -> io::Result<(SignalDriver, SignalHandle)> {
Ok((io_driver, ()))
}
}
@@ -78,10 +251,10 @@
// ===== process driver =====
cfg_process_driver! {
- type ProcessDriver = crate::process::unix::driver::Driver;
+ type ProcessDriver = crate::runtime::process::Driver;
fn create_process_driver(signal_driver: SignalDriver) -> ProcessDriver {
- crate::process::unix::driver::Driver::new(signal_driver)
+ ProcessDriver::new(signal_driver)
}
}
@@ -98,10 +271,16 @@
// ===== time driver =====
cfg_time! {
- type TimeDriver = crate::park::either::Either<crate::time::driver::Driver<IoStack>, IoStack>;
+ #[derive(Debug)]
+ pub(crate) enum TimeDriver {
+ Enabled {
+ driver: crate::runtime::time::Driver,
+ },
+ Disabled(IoStack),
+ }
pub(crate) type Clock = crate::time::Clock;
- pub(crate) type TimeHandle = Option<crate::time::driver::Handle>;
+ pub(crate) type TimeHandle = Option<crate::runtime::time::Handle>;
fn create_clock(enable_pausing: bool, start_paused: bool) -> Clock {
crate::time::Clock::new(enable_pausing, start_paused)
@@ -112,15 +291,35 @@
io_stack: IoStack,
clock: Clock,
) -> (TimeDriver, TimeHandle) {
- use crate::park::either::Either;
-
if enable {
- let driver = crate::time::driver::Driver::new(io_stack, clock);
- let handle = driver.handle();
+ let (driver, handle) = crate::runtime::time::Driver::new(io_stack, clock);
- (Either::A(driver), Some(handle))
+ (TimeDriver::Enabled { driver }, Some(handle))
} else {
- (Either::B(io_stack), None)
+ (TimeDriver::Disabled(io_stack), None)
+ }
+ }
+
+ impl TimeDriver {
+ pub(crate) fn park(&mut self, handle: &Handle) {
+ match self {
+ TimeDriver::Enabled { driver, .. } => driver.park(handle),
+ TimeDriver::Disabled(v) => v.park(handle),
+ }
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &Handle, duration: Duration) {
+ match self {
+ TimeDriver::Enabled { driver } => driver.park_timeout(handle, duration),
+ TimeDriver::Disabled(v) => v.park_timeout(handle, duration),
+ }
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &Handle) {
+ match self {
+ TimeDriver::Enabled { driver } => driver.shutdown(handle),
+ TimeDriver::Disabled(v) => v.shutdown(handle),
+ }
}
}
}
@@ -143,66 +342,3 @@
(io_stack, ())
}
}
-
-// ===== runtime driver =====
-
-#[derive(Debug)]
-pub(crate) struct Driver {
- inner: TimeDriver,
-}
-
-pub(crate) struct Resources {
- pub(crate) io_handle: IoHandle,
- pub(crate) signal_handle: SignalHandle,
- pub(crate) time_handle: TimeHandle,
- pub(crate) clock: Clock,
-}
-
-pub(crate) struct Cfg {
- pub(crate) enable_io: bool,
- pub(crate) enable_time: bool,
- pub(crate) enable_pause_time: bool,
- pub(crate) start_paused: bool,
-}
-
-impl Driver {
- pub(crate) fn new(cfg: Cfg) -> io::Result<(Self, Resources)> {
- let (io_stack, io_handle, signal_handle) = create_io_stack(cfg.enable_io)?;
-
- let clock = create_clock(cfg.enable_pause_time, cfg.start_paused);
-
- let (time_driver, time_handle) =
- create_time_driver(cfg.enable_time, io_stack, clock.clone());
-
- Ok((
- Self { inner: time_driver },
- Resources {
- io_handle,
- signal_handle,
- time_handle,
- clock,
- },
- ))
- }
-}
-
-impl Park for Driver {
- type Unpark = <TimeDriver as Park>::Unpark;
- type Error = <TimeDriver as Park>::Error;
-
- fn unpark(&self) -> Self::Unpark {
- self.inner.unpark()
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- self.inner.park()
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.inner.park_timeout(duration)
- }
-
- fn shutdown(&mut self) {
- self.inner.shutdown()
- }
-}
diff --git a/src/runtime/enter.rs b/src/runtime/enter.rs
deleted file mode 100644
index 3f14cb5..0000000
--- a/src/runtime/enter.rs
+++ /dev/null
@@ -1,205 +0,0 @@
-use std::cell::{Cell, RefCell};
-use std::fmt;
-use std::marker::PhantomData;
-
-#[derive(Debug, Clone, Copy)]
-pub(crate) enum EnterContext {
- #[cfg_attr(not(feature = "rt"), allow(dead_code))]
- Entered {
- allow_blocking: bool,
- },
- NotEntered,
-}
-
-impl EnterContext {
- pub(crate) fn is_entered(self) -> bool {
- matches!(self, EnterContext::Entered { .. })
- }
-}
-
-thread_local!(static ENTERED: Cell<EnterContext> = Cell::new(EnterContext::NotEntered));
-
-/// Represents an executor context.
-pub(crate) struct Enter {
- _p: PhantomData<RefCell<()>>,
-}
-
-cfg_rt! {
- use crate::park::thread::ParkError;
-
- use std::time::Duration;
-
- /// Marks the current thread as being within the dynamic extent of an
- /// executor.
- pub(crate) fn enter(allow_blocking: bool) -> Enter {
- if let Some(enter) = try_enter(allow_blocking) {
- return enter;
- }
-
- panic!(
- "Cannot start a runtime from within a runtime. This happens \
- because a function (like `block_on`) attempted to block the \
- current thread while the thread is being used to drive \
- asynchronous tasks."
- );
- }
-
- /// Tries to enter a runtime context, returns `None` if already in a runtime
- /// context.
- pub(crate) fn try_enter(allow_blocking: bool) -> Option<Enter> {
- ENTERED.with(|c| {
- if c.get().is_entered() {
- None
- } else {
- c.set(EnterContext::Entered { allow_blocking });
- Some(Enter { _p: PhantomData })
- }
- })
- }
-}
-
-// Forces the current "entered" state to be cleared while the closure
-// is executed.
-//
-// # Warning
-//
-// This is hidden for a reason. Do not use without fully understanding
-// executors. Misusing can easily cause your program to deadlock.
-cfg_rt_multi_thread! {
- pub(crate) fn exit<F: FnOnce() -> R, R>(f: F) -> R {
- // Reset in case the closure panics
- struct Reset(EnterContext);
- impl Drop for Reset {
- fn drop(&mut self) {
- ENTERED.with(|c| {
- assert!(!c.get().is_entered(), "closure claimed permanent executor");
- c.set(self.0);
- });
- }
- }
-
- let was = ENTERED.with(|c| {
- let e = c.get();
- assert!(e.is_entered(), "asked to exit when not entered");
- c.set(EnterContext::NotEntered);
- e
- });
-
- let _reset = Reset(was);
- // dropping _reset after f() will reset ENTERED
- f()
- }
-}
-
-cfg_rt! {
- /// Disallows blocking in the current runtime context until the guard is dropped.
- pub(crate) fn disallow_blocking() -> DisallowBlockingGuard {
- let reset = ENTERED.with(|c| {
- if let EnterContext::Entered {
- allow_blocking: true,
- } = c.get()
- {
- c.set(EnterContext::Entered {
- allow_blocking: false,
- });
- true
- } else {
- false
- }
- });
- DisallowBlockingGuard(reset)
- }
-
- pub(crate) struct DisallowBlockingGuard(bool);
- impl Drop for DisallowBlockingGuard {
- fn drop(&mut self) {
- if self.0 {
- // XXX: Do we want some kind of assertion here, or is "best effort" okay?
- ENTERED.with(|c| {
- if let EnterContext::Entered {
- allow_blocking: false,
- } = c.get()
- {
- c.set(EnterContext::Entered {
- allow_blocking: true,
- });
- }
- })
- }
- }
- }
-}
-
-cfg_rt_multi_thread! {
- /// Returns true if in a runtime context.
- pub(crate) fn context() -> EnterContext {
- ENTERED.with(|c| c.get())
- }
-}
-
-cfg_rt! {
- impl Enter {
- /// Blocks the thread on the specified future, returning the value with
- /// which that future completes.
- pub(crate) fn block_on<F>(&mut self, f: F) -> Result<F::Output, ParkError>
- where
- F: std::future::Future,
- {
- use crate::park::thread::CachedParkThread;
-
- let mut park = CachedParkThread::new();
- park.block_on(f)
- }
-
- /// Blocks the thread on the specified future for **at most** `timeout`
- ///
- /// If the future completes before `timeout`, the result is returned. If
- /// `timeout` elapses, then `Err` is returned.
- pub(crate) fn block_on_timeout<F>(&mut self, f: F, timeout: Duration) -> Result<F::Output, ParkError>
- where
- F: std::future::Future,
- {
- use crate::park::Park;
- use crate::park::thread::CachedParkThread;
- use std::task::Context;
- use std::task::Poll::Ready;
- use std::time::Instant;
-
- let mut park = CachedParkThread::new();
- let waker = park.get_unpark()?.into_waker();
- let mut cx = Context::from_waker(&waker);
-
- pin!(f);
- let when = Instant::now() + timeout;
-
- loop {
- if let Ready(v) = crate::coop::budget(|| f.as_mut().poll(&mut cx)) {
- return Ok(v);
- }
-
- let now = Instant::now();
-
- if now >= when {
- return Err(());
- }
-
- park.park_timeout(when - now)?;
- }
- }
- }
-}
-
-impl fmt::Debug for Enter {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_struct("Enter").finish()
- }
-}
-
-impl Drop for Enter {
- fn drop(&mut self) {
- ENTERED.with(|c| {
- assert!(c.get().is_entered());
- c.set(EnterContext::NotEntered);
- });
- }
-}
diff --git a/src/runtime/handle.rs b/src/runtime/handle.rs
index cd1cb76..c5dc65f 100644
--- a/src/runtime/handle.rs
+++ b/src/runtime/handle.rs
@@ -1,11 +1,4 @@
-use crate::runtime::blocking::{BlockingTask, NoopSchedule};
-use crate::runtime::task::{self, JoinHandle};
-use crate::runtime::{blocking, context, driver, Spawner};
-use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR};
-
-use std::future::Future;
-use std::marker::PhantomData;
-use std::{error, fmt};
+use crate::runtime::{context, scheduler, RuntimeFlavor};
/// Handle to the runtime.
///
@@ -14,35 +7,19 @@
///
/// [`Runtime::handle`]: crate::runtime::Runtime::handle()
#[derive(Debug, Clone)]
+// When the `rt` feature is *not* enabled, this type is still defined, but not
+// included in the public API.
pub struct Handle {
- pub(super) spawner: Spawner,
-
- /// Handles to the I/O drivers
- #[cfg_attr(
- not(any(feature = "net", feature = "process", all(unix, feature = "signal"))),
- allow(dead_code)
- )]
- pub(super) io_handle: driver::IoHandle,
-
- /// Handles to the signal drivers
- #[cfg_attr(
- not(any(feature = "signal", all(unix, feature = "process"))),
- allow(dead_code)
- )]
- pub(super) signal_handle: driver::SignalHandle,
-
- /// Handles to the time drivers
- #[cfg_attr(not(feature = "time"), allow(dead_code))]
- pub(super) time_handle: driver::TimeHandle,
-
- /// Source of `Instant::now()`
- #[cfg_attr(not(all(feature = "time", feature = "test-util")), allow(dead_code))]
- pub(super) clock: driver::Clock,
-
- /// Blocking pool spawner
- pub(super) blocking_spawner: blocking::Spawner,
+ pub(crate) inner: scheduler::Handle,
}
+use crate::runtime::task::JoinHandle;
+use crate::util::error::{CONTEXT_MISSING_ERROR, THREAD_LOCAL_DESTROYED_ERROR};
+
+use std::future::Future;
+use std::marker::PhantomData;
+use std::{error, fmt};
+
/// Runtime context guard.
///
/// Returned by [`Runtime::enter`] and [`Handle::enter`], the context guard exits
@@ -52,32 +29,37 @@
#[derive(Debug)]
#[must_use = "Creating and dropping a guard does nothing"]
pub struct EnterGuard<'a> {
- _guard: context::EnterGuard,
+ _guard: context::SetCurrentGuard,
_handle_lifetime: PhantomData<&'a Handle>,
}
impl Handle {
/// Enters the runtime context. This allows you to construct types that must
/// have an executor available on creation such as [`Sleep`] or [`TcpStream`].
- /// It will also allow you to call methods such as [`tokio::spawn`].
+ /// It will also allow you to call methods such as [`tokio::spawn`] and [`Handle::current`]
+ /// without panicking.
///
/// [`Sleep`]: struct@crate::time::Sleep
/// [`TcpStream`]: struct@crate::net::TcpStream
/// [`tokio::spawn`]: fn@crate::spawn
pub fn enter(&self) -> EnterGuard<'_> {
EnterGuard {
- _guard: context::enter(self.clone()),
+ _guard: match context::try_set_current(&self.inner) {
+ Some(guard) => guard,
+ None => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
+ },
_handle_lifetime: PhantomData,
}
}
/// Returns a `Handle` view over the currently running `Runtime`.
///
- /// # Panic
+ /// # Panics
///
/// This will panic if called outside the context of a Tokio runtime. That means that you must
- /// call this on one of the threads **being run by the runtime**. Calling this from within a
- /// thread created by `std::thread::spawn` (for example) will cause a panic.
+ /// call this on one of the threads **being run by the runtime**, or from a thread with an active
+ /// `EnterGuard`. Calling this from within a thread created by `std::thread::spawn` (for example)
+ /// will cause a panic unless that thread has an active `EnterGuard`.
///
/// # Examples
///
@@ -101,16 +83,24 @@
/// # let handle =
/// thread::spawn(move || {
/// // Notice that the handle is created outside of this thread and then moved in
- /// handle.spawn(async { /* ... */ })
- /// // This next line would cause a panic
- /// // let handle2 = Handle::current();
+ /// handle.spawn(async { /* ... */ });
+ /// // This next line would cause a panic because we haven't entered the runtime
+ /// // and created an EnterGuard
+ /// // let handle2 = Handle::current(); // panic
+ /// // So we create a guard here with Handle::enter();
+ /// let _guard = handle.enter();
+ /// // Now we can call Handle::current();
+ /// let handle2 = Handle::current();
/// });
/// # handle.join().unwrap();
/// # });
/// # }
/// ```
+ #[track_caller]
pub fn current() -> Self {
- context::current()
+ Handle {
+ inner: scheduler::Handle::current(),
+ }
}
/// Returns a Handle view over the currently running Runtime
@@ -119,15 +109,7 @@
///
/// Contrary to `current`, this never panics
pub fn try_current() -> Result<Self, TryCurrentError> {
- context::try_current()
- }
-
- cfg_stats! {
- /// Returns a view that lets you get information about how the runtime
- /// is performing.
- pub fn stats(&self) -> &crate::runtime::stats::RuntimeStats {
- self.spawner.stats()
- }
+ context::try_current().map(|inner| Handle { inner })
}
/// Spawns a future onto the Tokio runtime.
@@ -136,6 +118,10 @@
/// thread pool. The thread pool is then responsible for polling the future
/// until it completes.
///
+ /// The provided future will start running in the background immediately
+ /// when `spawn` is called, even if you don't await the returned
+ /// `JoinHandle`.
+ ///
/// See [module level][mod] documentation for more details.
///
/// [mod]: index.html
@@ -157,15 +143,13 @@
/// });
/// # }
/// ```
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
where
F: Future + Send + 'static,
F::Output: Send + 'static,
{
- #[cfg(all(tokio_unstable, feature = "tracing"))]
- let future = crate::util::trace::task(future, "task", None);
- self.spawner.spawn(future)
+ self.spawn_named(future, None)
}
/// Runs the provided function on an executor dedicated to blocking.
@@ -187,58 +171,13 @@
/// println!("now running on a worker thread");
/// });
/// # }
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
where
F: FnOnce() -> R + Send + 'static,
R: Send + 'static,
{
- if cfg!(debug_assertions) && std::mem::size_of::<F>() > 2048 {
- self.spawn_blocking_inner(Box::new(func), None)
- } else {
- self.spawn_blocking_inner(func, None)
- }
- }
-
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub(crate) fn spawn_blocking_inner<F, R>(&self, func: F, name: Option<&str>) -> JoinHandle<R>
- where
- F: FnOnce() -> R + Send + 'static,
- R: Send + 'static,
- {
- let fut = BlockingTask::new(func);
-
- #[cfg(all(tokio_unstable, feature = "tracing"))]
- let fut = {
- use tracing::Instrument;
- #[cfg(tokio_track_caller)]
- let location = std::panic::Location::caller();
- #[cfg(tokio_track_caller)]
- let span = tracing::trace_span!(
- target: "tokio::task::blocking",
- "runtime.spawn",
- kind = %"blocking",
- task.name = %name.unwrap_or_default(),
- "fn" = %std::any::type_name::<F>(),
- spawn.location = %format_args!("{}:{}:{}", location.file(), location.line(), location.column()),
- );
- #[cfg(not(tokio_track_caller))]
- let span = tracing::trace_span!(
- target: "tokio::task::blocking",
- "runtime.spawn",
- kind = %"blocking",
- task.name = %name.unwrap_or_default(),
- "fn" = %std::any::type_name::<F>(),
- );
- fut.instrument(span)
- };
-
- #[cfg(not(all(tokio_unstable, feature = "tracing")))]
- let _ = name;
-
- let (task, handle) = task::unowned(fut, NoopSchedule);
- let _ = self.blocking_spawner.spawn(task, self);
- handle
+ self.inner.blocking_spawner().spawn_blocking(self, func)
}
/// Runs a future to completion on this `Handle`'s associated `Runtime`.
@@ -311,25 +250,74 @@
/// [`tokio::fs`]: crate::fs
/// [`tokio::net`]: crate::net
/// [`tokio::time`]: crate::time
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub fn block_on<F: Future>(&self, future: F) -> F::Output {
#[cfg(all(tokio_unstable, feature = "tracing"))]
- let future = crate::util::trace::task(future, "block_on", None);
+ let future =
+ crate::util::trace::task(future, "block_on", None, super::task::Id::next().as_u64());
- // Enter the **runtime** context. This configures spawning, the current I/O driver, ...
- let _rt_enter = self.enter();
-
- // Enter a **blocking** context. This prevents blocking from a runtime.
- let mut blocking_enter = crate::runtime::enter(true);
+ // Enter the runtime context. This sets the current driver handles and
+ // prevents blocking an existing runtime.
+ let mut enter = context::enter_runtime(&self.inner, true);
// Block on the future
- blocking_enter
+ enter
+ .blocking
.block_on(future)
.expect("failed to park thread")
}
- pub(crate) fn shutdown(mut self) {
- self.spawner.shutdown();
+ #[track_caller]
+ pub(crate) fn spawn_named<F>(&self, future: F, _name: Option<&str>) -> JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ let id = crate::runtime::task::Id::next();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let future = crate::util::trace::task(future, "task", _name, id.as_u64());
+ self.inner.spawn(future, id)
+ }
+
+ /// Returns the flavor of the current `Runtime`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::{Handle, RuntimeFlavor};
+ ///
+ /// #[tokio::main(flavor = "current_thread")]
+ /// async fn main() {
+ /// assert_eq!(RuntimeFlavor::CurrentThread, Handle::current().runtime_flavor());
+ /// }
+ /// ```
+ ///
+ /// ```
+ /// use tokio::runtime::{Handle, RuntimeFlavor};
+ ///
+ /// #[tokio::main(flavor = "multi_thread", worker_threads = 4)]
+ /// async fn main() {
+ /// assert_eq!(RuntimeFlavor::MultiThread, Handle::current().runtime_flavor());
+ /// }
+ /// ```
+ pub fn runtime_flavor(&self) -> RuntimeFlavor {
+ match self.inner {
+ scheduler::Handle::CurrentThread(_) => RuntimeFlavor::CurrentThread,
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ scheduler::Handle::MultiThread(_) => RuntimeFlavor::MultiThread,
+ }
+ }
+}
+
+cfg_metrics! {
+ use crate::runtime::RuntimeMetrics;
+
+ impl Handle {
+ /// Returns a view that lets you get information about how the runtime
+ /// is performing.
+ pub fn metrics(&self) -> RuntimeMetrics {
+ RuntimeMetrics::new(self.clone())
+ }
}
}
diff --git a/src/runtime/io/metrics.rs b/src/runtime/io/metrics.rs
new file mode 100644
index 0000000..ec341ef
--- /dev/null
+++ b/src/runtime/io/metrics.rs
@@ -0,0 +1,24 @@
+//! This file contains mocks of the metrics types used in the I/O driver.
+//!
+//! The reason these mocks don't live in `src/runtime/mock.rs` is because
+//! these need to be available in the case when `net` is enabled but
+//! `rt` is not.
+
+cfg_not_rt_and_metrics_and_net! {
+ #[derive(Default)]
+ pub(crate) struct IoDriverMetrics {}
+
+ impl IoDriverMetrics {
+ pub(crate) fn incr_fd_count(&self) {}
+ pub(crate) fn dec_fd_count(&self) {}
+ pub(crate) fn incr_ready_count_by(&self, _amt: u64) {}
+ }
+}
+
+cfg_net! {
+ cfg_rt! {
+ cfg_metrics! {
+ pub(crate) use crate::runtime::IoDriverMetrics;
+ }
+ }
+}
diff --git a/src/runtime/io/mod.rs b/src/runtime/io/mod.rs
new file mode 100644
index 0000000..2e578b6
--- /dev/null
+++ b/src/runtime/io/mod.rs
@@ -0,0 +1,344 @@
+#![cfg_attr(not(all(feature = "rt", feature = "net")), allow(dead_code))]
+
+mod registration;
+pub(crate) use registration::Registration;
+
+mod scheduled_io;
+use scheduled_io::ScheduledIo;
+
+mod metrics;
+
+use crate::io::interest::Interest;
+use crate::io::ready::Ready;
+use crate::runtime::driver;
+use crate::util::slab::{self, Slab};
+use crate::{loom::sync::RwLock, util::bit};
+
+use metrics::IoDriverMetrics;
+
+use std::fmt;
+use std::io;
+use std::time::Duration;
+
+/// I/O driver, backed by Mio.
+pub(crate) struct Driver {
+ /// Tracks the number of times `turn` is called. It is safe for this to wrap
+ /// as it is mostly used to determine when to call `compact()`.
+ tick: u8,
+
+ /// True when an event with the signal token is received
+ signal_ready: bool,
+
+ /// Reuse the `mio::Events` value across calls to poll.
+ events: mio::Events,
+
+ /// Primary slab handle containing the state for each resource registered
+ /// with this driver.
+ resources: Slab<ScheduledIo>,
+
+ /// The system event queue.
+ poll: mio::Poll,
+}
+
+/// A reference to an I/O driver.
+pub(crate) struct Handle {
+ /// Registers I/O resources.
+ registry: mio::Registry,
+
+ /// Allocates `ScheduledIo` handles when creating new resources.
+ io_dispatch: RwLock<IoDispatcher>,
+
+ /// Used to wake up the reactor from a call to `turn`.
+ /// Not supported on Wasi due to lack of threading support.
+ #[cfg(not(tokio_wasi))]
+ waker: mio::Waker,
+
+ pub(crate) metrics: IoDriverMetrics,
+}
+
+#[derive(Debug)]
+pub(crate) struct ReadyEvent {
+ tick: u8,
+ pub(crate) ready: Ready,
+ is_shutdown: bool,
+}
+
+struct IoDispatcher {
+ allocator: slab::Allocator<ScheduledIo>,
+ is_shutdown: bool,
+}
+
+#[derive(Debug, Eq, PartialEq, Clone, Copy)]
+enum Direction {
+ Read,
+ Write,
+}
+
+enum Tick {
+ Set(u8),
+ Clear(u8),
+}
+
+// TODO: Don't use a fake token. Instead, reserve a slot entry for the wakeup
+// token.
+const TOKEN_WAKEUP: mio::Token = mio::Token(1 << 31);
+const TOKEN_SIGNAL: mio::Token = mio::Token(1 + (1 << 31));
+
+const ADDRESS: bit::Pack = bit::Pack::least_significant(24);
+
+// Packs the generation value in the `readiness` field.
+//
+// The generation prevents a race condition where a slab slot is reused for a
+// new socket while the I/O driver is about to apply a readiness event. The
+// generation value is checked when setting new readiness. If the generation do
+// not match, then the readiness event is discarded.
+const GENERATION: bit::Pack = ADDRESS.then(7);
+
+fn _assert_kinds() {
+ fn _assert<T: Send + Sync>() {}
+
+ _assert::<Handle>();
+}
+
+// ===== impl Driver =====
+
+impl Driver {
+ /// Creates a new event loop, returning any error that happened during the
+ /// creation.
+ pub(crate) fn new(nevents: usize) -> io::Result<(Driver, Handle)> {
+ let poll = mio::Poll::new()?;
+ #[cfg(not(tokio_wasi))]
+ let waker = mio::Waker::new(poll.registry(), TOKEN_WAKEUP)?;
+ let registry = poll.registry().try_clone()?;
+
+ let slab = Slab::new();
+ let allocator = slab.allocator();
+
+ let driver = Driver {
+ tick: 0,
+ signal_ready: false,
+ events: mio::Events::with_capacity(nevents),
+ poll,
+ resources: slab,
+ };
+
+ let handle = Handle {
+ registry,
+ io_dispatch: RwLock::new(IoDispatcher::new(allocator)),
+ #[cfg(not(tokio_wasi))]
+ waker,
+ metrics: IoDriverMetrics::default(),
+ };
+
+ Ok((driver, handle))
+ }
+
+ pub(crate) fn park(&mut self, rt_handle: &driver::Handle) {
+ let handle = rt_handle.io();
+ self.turn(handle, None);
+ }
+
+ pub(crate) fn park_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) {
+ let handle = rt_handle.io();
+ self.turn(handle, Some(duration));
+ }
+
+ pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) {
+ let handle = rt_handle.io();
+
+ if handle.shutdown() {
+ self.resources.for_each(|io| {
+ // If a task is waiting on the I/O resource, notify it that the
+ // runtime is being shutdown. And shutdown will clear all wakers.
+ io.shutdown();
+ });
+ }
+ }
+
+ fn turn(&mut self, handle: &Handle, max_wait: Option<Duration>) {
+ // How often to call `compact()` on the resource slab
+ const COMPACT_INTERVAL: u8 = 255;
+
+ self.tick = self.tick.wrapping_add(1);
+
+ if self.tick == COMPACT_INTERVAL {
+ self.resources.compact()
+ }
+
+ let events = &mut self.events;
+
+ // Block waiting for an event to happen, peeling out how many events
+ // happened.
+ match self.poll.poll(events, max_wait) {
+ Ok(_) => {}
+ Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
+ #[cfg(tokio_wasi)]
+ Err(e) if e.kind() == io::ErrorKind::InvalidInput => {
+ // In case of wasm32_wasi this error happens, when trying to poll without subscriptions
+ // just return from the park, as there would be nothing, which wakes us up.
+ }
+ Err(e) => panic!("unexpected error when polling the I/O driver: {:?}", e),
+ }
+
+ // Process all the events that came in, dispatching appropriately
+ let mut ready_count = 0;
+ for event in events.iter() {
+ let token = event.token();
+
+ if token == TOKEN_WAKEUP {
+ // Nothing to do, the event is used to unblock the I/O driver
+ } else if token == TOKEN_SIGNAL {
+ self.signal_ready = true;
+ } else {
+ Self::dispatch(
+ &mut self.resources,
+ self.tick,
+ token,
+ Ready::from_mio(event),
+ );
+ ready_count += 1;
+ }
+ }
+
+ handle.metrics.incr_ready_count_by(ready_count);
+ }
+
+ fn dispatch(resources: &mut Slab<ScheduledIo>, tick: u8, token: mio::Token, ready: Ready) {
+ let addr = slab::Address::from_usize(ADDRESS.unpack(token.0));
+
+ let io = match resources.get(addr) {
+ Some(io) => io,
+ None => return,
+ };
+
+ let res = io.set_readiness(Some(token.0), Tick::Set(tick), |curr| curr | ready);
+
+ if res.is_err() {
+ // token no longer valid!
+ return;
+ }
+
+ io.wake(ready);
+ }
+}
+
+impl fmt::Debug for Driver {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "Driver")
+ }
+}
+
+impl Handle {
+ /// Forces a reactor blocked in a call to `turn` to wakeup, or otherwise
+ /// makes the next call to `turn` return immediately.
+ ///
+ /// This method is intended to be used in situations where a notification
+ /// needs to otherwise be sent to the main reactor. If the reactor is
+ /// currently blocked inside of `turn` then it will wake up and soon return
+ /// after this method has been called. If the reactor is not currently
+ /// blocked in `turn`, then the next call to `turn` will not block and
+ /// return immediately.
+ pub(crate) fn unpark(&self) {
+ #[cfg(not(tokio_wasi))]
+ self.waker.wake().expect("failed to wake I/O driver");
+ }
+
+ /// Registers an I/O resource with the reactor for a given `mio::Ready` state.
+ ///
+ /// The registration token is returned.
+ pub(super) fn add_source(
+ &self,
+ source: &mut impl mio::event::Source,
+ interest: Interest,
+ ) -> io::Result<slab::Ref<ScheduledIo>> {
+ let (address, shared) = self.allocate()?;
+
+ let token = GENERATION.pack(shared.generation(), ADDRESS.pack(address.as_usize(), 0));
+
+ self.registry
+ .register(source, mio::Token(token), interest.to_mio())?;
+
+ self.metrics.incr_fd_count();
+
+ Ok(shared)
+ }
+
+ /// Deregisters an I/O resource from the reactor.
+ pub(super) fn deregister_source(&self, source: &mut impl mio::event::Source) -> io::Result<()> {
+ self.registry.deregister(source)?;
+
+ self.metrics.dec_fd_count();
+
+ Ok(())
+ }
+
+ /// shutdown the dispatcher.
+ fn shutdown(&self) -> bool {
+ let mut io = self.io_dispatch.write().unwrap();
+ if io.is_shutdown {
+ return false;
+ }
+ io.is_shutdown = true;
+ true
+ }
+
+ fn allocate(&self) -> io::Result<(slab::Address, slab::Ref<ScheduledIo>)> {
+ let io = self.io_dispatch.read().unwrap();
+ if io.is_shutdown {
+ return Err(io::Error::new(
+ io::ErrorKind::Other,
+ crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
+ ));
+ }
+ io.allocator.allocate().ok_or_else(|| {
+ io::Error::new(
+ io::ErrorKind::Other,
+ "reactor at max registered I/O resources",
+ )
+ })
+ }
+}
+
+impl fmt::Debug for Handle {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "Handle")
+ }
+}
+
+// ===== impl IoDispatcher =====
+
+impl IoDispatcher {
+ fn new(allocator: slab::Allocator<ScheduledIo>) -> Self {
+ Self {
+ allocator,
+ is_shutdown: false,
+ }
+ }
+}
+
+impl Direction {
+ pub(super) fn mask(self) -> Ready {
+ match self {
+ Direction::Read => Ready::READABLE | Ready::READ_CLOSED,
+ Direction::Write => Ready::WRITABLE | Ready::WRITE_CLOSED,
+ }
+ }
+}
+
+// Signal handling
+cfg_signal_internal_and_unix! {
+ impl Handle {
+ pub(crate) fn register_signal_receiver(&self, receiver: &mut mio::net::UnixStream) -> io::Result<()> {
+ self.registry.register(receiver, TOKEN_SIGNAL, mio::Interest::READABLE)?;
+ Ok(())
+ }
+ }
+
+ impl Driver {
+ pub(crate) fn consume_signal_ready(&mut self) -> bool {
+ let ret = self.signal_ready;
+ self.signal_ready = false;
+ ret
+ }
+ }
+}
diff --git a/src/io/driver/platform.rs b/src/runtime/io/platform.rs
similarity index 100%
rename from src/io/driver/platform.rs
rename to src/runtime/io/platform.rs
diff --git a/src/io/driver/registration.rs b/src/runtime/io/registration.rs
similarity index 82%
rename from src/io/driver/registration.rs
rename to src/runtime/io/registration.rs
index 7350be6..140b924 100644
--- a/src/io/driver/registration.rs
+++ b/src/runtime/io/registration.rs
@@ -1,6 +1,8 @@
#![cfg_attr(not(feature = "net"), allow(dead_code))]
-use crate::io::driver::{Direction, Handle, Interest, ReadyEvent, ScheduledIo};
+use crate::io::interest::Interest;
+use crate::runtime::io::{Direction, Handle, ReadyEvent, ScheduledIo};
+use crate::runtime::scheduler;
use crate::util::slab;
use mio::event::Source;
@@ -42,8 +44,8 @@
/// [`poll_write_ready`]: method@Self::poll_write_ready`
#[derive(Debug)]
pub(crate) struct Registration {
- /// Handle to the associated driver.
- handle: Handle,
+ /// Handle to the associated runtime.
+ handle: scheduler::Handle,
/// Reference to state stored by the driver.
shared: slab::Ref<ScheduledIo>,
@@ -56,10 +58,8 @@
// ===== impl Registration =====
impl Registration {
- /// Registers the I/O resource with the default reactor, for a specific
- /// `Interest`. `new_with_interest` should be used over `new` when you need
- /// control over the readiness state, such as when a file descriptor only
- /// allows reads. This does not add `hup` or `error` so if you are
+ /// Registers the I/O resource with the reactor for the provided handle, for
+ /// a specific `Interest`. This does not add `hup` or `error` so if you are
/// interested in those states, you will need to add them to the readiness
/// state passed to this function.
///
@@ -67,19 +67,13 @@
///
/// - `Ok` if the registration happened successfully
/// - `Err` if an error was encountered during registration
+ #[track_caller]
pub(crate) fn new_with_interest_and_handle(
io: &mut impl Source,
interest: Interest,
- handle: Handle,
+ handle: scheduler::Handle,
) -> io::Result<Registration> {
- let shared = if let Some(inner) = handle.inner() {
- inner.add_source(io, interest)?
- } else {
- return Err(io::Error::new(
- io::ErrorKind::Other,
- "failed to find event loop",
- ));
- };
+ let shared = handle.driver().io().add_source(io, interest)?;
Ok(Registration { handle, shared })
}
@@ -101,11 +95,7 @@
///
/// `Err` is returned if an error is encountered.
pub(crate) fn deregister(&mut self, io: &mut impl Source) -> io::Result<()> {
- let inner = match self.handle.inner() {
- Some(inner) => inner,
- None => return Err(io::Error::new(io::ErrorKind::Other, "reactor gone")),
- };
- inner.deregister_source(io)
+ self.handle().deregister_source(io)
}
pub(crate) fn clear_readiness(&self, event: ReadyEvent) {
@@ -126,6 +116,7 @@
// Uses the poll path, requiring the caller to ensure mutual exclusion for
// correctness. Only the last task to call this function is notified.
+ #[cfg(not(tokio_wasi))]
pub(crate) fn poll_read_io<R>(
&self,
cx: &mut Context<'_>,
@@ -154,10 +145,10 @@
direction: Direction,
) -> Poll<io::Result<ReadyEvent>> {
// Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
let ev = ready!(self.shared.poll_readiness(cx, direction));
- if self.handle.inner().is_none() {
+ if ev.is_shutdown {
return Poll::Ready(Err(gone()));
}
@@ -206,6 +197,10 @@
res => res,
}
}
+
+ fn handle(&self) -> &Handle {
+ self.handle.driver().io()
+ }
}
impl Drop for Registration {
@@ -222,28 +217,22 @@
}
fn gone() -> io::Error {
- io::Error::new(io::ErrorKind::Other, "IO driver has terminated")
+ io::Error::new(
+ io::ErrorKind::Other,
+ crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR,
+ )
}
cfg_io_readiness! {
impl Registration {
pub(crate) async fn readiness(&self, interest: Interest) -> io::Result<ReadyEvent> {
- use std::future::Future;
- use std::pin::Pin;
+ let ev = self.shared.readiness(interest).await;
- let fut = self.shared.readiness(interest);
- pin!(fut);
+ if ev.is_shutdown {
+ return Err(gone())
+ }
- crate::future::poll_fn(|cx| {
- if self.handle.inner().is_none() {
- return Poll::Ready(Err(io::Error::new(
- io::ErrorKind::Other,
- crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR
- )));
- }
-
- Pin::new(&mut fut).poll(cx).map(Ok)
- }).await
+ Ok(ev)
}
pub(crate) async fn async_io<R>(&self, interest: Interest, mut f: impl FnMut() -> io::Result<R>) -> io::Result<R> {
diff --git a/src/io/driver/scheduled_io.rs b/src/runtime/io/scheduled_io.rs
similarity index 87%
rename from src/io/driver/scheduled_io.rs
rename to src/runtime/io/scheduled_io.rs
index 76f9343..197a4e0 100644
--- a/src/io/driver/scheduled_io.rs
+++ b/src/runtime/io/scheduled_io.rs
@@ -1,4 +1,6 @@
-use super::{Interest, Ready, ReadyEvent, Tick};
+use super::{ReadyEvent, Tick};
+use crate::io::interest::Interest;
+use crate::io::ready::Ready;
use crate::loom::sync::atomic::AtomicUsize;
use crate::loom::sync::Mutex;
use crate::util::bit;
@@ -44,9 +46,6 @@
/// Waker used for AsyncWrite.
writer: Option<Waker>,
-
- /// True if this ScheduledIo has been killed due to IO driver shutdown.
- is_shutdown: bool,
}
cfg_io_readiness! {
@@ -66,6 +65,14 @@
_p: PhantomPinned,
}
+ generate_addr_of_methods! {
+ impl<> Waiter {
+ unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> {
+ &self.pointers
+ }
+ }
+ }
+
/// Future returned by `readiness()`.
struct Readiness<'a> {
scheduled_io: &'a ScheduledIo,
@@ -85,7 +92,7 @@
// The `ScheduledIo::readiness` (`AtomicUsize`) is packed full of goodness.
//
-// | reserved | generation | driver tick | readiness |
+// | shutdown | generation | driver tick | readiness |
// |----------+------------+--------------+-----------|
// | 1 bit | 7 bits + 8 bits + 16 bits |
@@ -95,6 +102,8 @@
const GENERATION: bit::Pack = TICK.then(7);
+const SHUTDOWN: bit::Pack = GENERATION.then(1);
+
#[test]
fn test_generations_assert_same() {
assert_eq!(super::GENERATION, GENERATION);
@@ -128,9 +137,11 @@
}
/// Invoked when the IO driver is shut down; forces this ScheduledIo into a
- /// permanently ready state.
+ /// permanently shutdown state.
pub(super) fn shutdown(&self) {
- self.wake0(Ready::ALL, true)
+ let mask = SHUTDOWN.pack(1, 0);
+ self.readiness.fetch_or(mask, AcqRel);
+ self.wake(Ready::ALL);
}
/// Sets the readiness on this `ScheduledIo` by invoking the given closure on
@@ -209,16 +220,10 @@
/// than 32 wakers to notify, if the stack array fills up, the lock is
/// released, the array is cleared, and the iteration continues.
pub(super) fn wake(&self, ready: Ready) {
- self.wake0(ready, false);
- }
-
- fn wake0(&self, ready: Ready, shutdown: bool) {
let mut wakers = WakeList::new();
let mut waiters = self.waiters.lock();
- waiters.is_shutdown |= shutdown;
-
// check for AsyncRead slot
if ready.is_readable() {
if let Some(waker) = waiters.reader.take() {
@@ -273,6 +278,7 @@
ReadyEvent {
tick: TICK.unpack(curr) as u8,
ready: interest.mask() & Ready::from_usize(READINESS.unpack(curr)),
+ is_shutdown: SHUTDOWN.unpack(curr) != 0,
}
}
@@ -289,8 +295,9 @@
let curr = self.readiness.load(Acquire);
let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr));
+ let is_shutdown = SHUTDOWN.unpack(curr) != 0;
- if ready.is_empty() {
+ if ready.is_empty() && !is_shutdown {
// Update the task info
let mut waiters = self.waiters.lock();
let slot = match direction {
@@ -315,10 +322,12 @@
// taking the waiters lock
let curr = self.readiness.load(Acquire);
let ready = direction.mask() & Ready::from_usize(READINESS.unpack(curr));
- if waiters.is_shutdown {
+ let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+ if is_shutdown {
Poll::Ready(ReadyEvent {
tick: TICK.unpack(curr) as u8,
ready: direction.mask(),
+ is_shutdown,
})
} else if ready.is_empty() {
Poll::Pending
@@ -326,12 +335,14 @@
Poll::Ready(ReadyEvent {
tick: TICK.unpack(curr) as u8,
ready,
+ is_shutdown,
})
}
} else {
Poll::Ready(ReadyEvent {
tick: TICK.unpack(curr) as u8,
ready,
+ is_shutdown,
})
}
}
@@ -399,8 +410,8 @@
ptr
}
- unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
- NonNull::from(&mut target.as_mut().pointers)
+ unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
+ Waiter::addr_of_pointers(target)
}
}
@@ -423,16 +434,17 @@
// Optimistically check existing readiness
let curr = scheduled_io.readiness.load(SeqCst);
let ready = Ready::from_usize(READINESS.unpack(curr));
+ let is_shutdown = SHUTDOWN.unpack(curr) != 0;
// Safety: `waiter.interest` never changes
let interest = unsafe { (*waiter.get()).interest };
let ready = ready.intersection(interest);
- if !ready.is_empty() {
+ if !ready.is_empty() || is_shutdown {
// Currently ready!
let tick = TICK.unpack(curr) as u8;
*state = State::Done;
- return Poll::Ready(ReadyEvent { tick, ready });
+ return Poll::Ready(ReadyEvent { tick, ready, is_shutdown });
}
// Wasn't ready, take the lock (and check again while locked).
@@ -440,18 +452,19 @@
let curr = scheduled_io.readiness.load(SeqCst);
let mut ready = Ready::from_usize(READINESS.unpack(curr));
+ let is_shutdown = SHUTDOWN.unpack(curr) != 0;
- if waiters.is_shutdown {
+ if is_shutdown {
ready = Ready::ALL;
}
let ready = ready.intersection(interest);
- if !ready.is_empty() {
+ if !ready.is_empty() || is_shutdown {
// Currently ready!
let tick = TICK.unpack(curr) as u8;
*state = State::Done;
- return Poll::Ready(ReadyEvent { tick, ready });
+ return Poll::Ready(ReadyEvent { tick, ready, is_shutdown });
}
// Not ready even after locked, insert into list...
@@ -500,14 +513,26 @@
drop(waiters);
}
State::Done => {
- let tick = TICK.unpack(scheduled_io.readiness.load(Acquire)) as u8;
-
// Safety: State::Done means it is no longer shared
let w = unsafe { &mut *waiter.get() };
+ let curr = scheduled_io.readiness.load(Acquire);
+ let is_shutdown = SHUTDOWN.unpack(curr) != 0;
+
+ // The returned tick might be newer than the event
+ // which notified our waker. This is ok because the future
+ // still didn't return `Poll::Ready`.
+ let tick = TICK.unpack(curr) as u8;
+
+ // The readiness state could have been cleared in the meantime,
+ // but we allow the returned ready set to be empty.
+ let curr_ready = Ready::from_usize(READINESS.unpack(curr));
+ let ready = curr_ready.intersection(w.interest);
+
return Poll::Ready(ReadyEvent {
tick,
- ready: Ready::from_interest(w.interest),
+ ready,
+ is_shutdown,
});
}
}
diff --git a/src/runtime/metrics/batch.rs b/src/runtime/metrics/batch.rs
new file mode 100644
index 0000000..4e6b28d
--- /dev/null
+++ b/src/runtime/metrics/batch.rs
@@ -0,0 +1,116 @@
+use crate::runtime::WorkerMetrics;
+
+use std::convert::TryFrom;
+use std::sync::atomic::Ordering::Relaxed;
+use std::time::Instant;
+
+pub(crate) struct MetricsBatch {
+ /// Number of times the worker parked.
+ park_count: u64,
+
+ /// Number of times the worker woke w/o doing work.
+ noop_count: u64,
+
+ /// Number of tasks stolen.
+ steal_count: u64,
+
+ /// Number of times tasks where stolen.
+ steal_operations: u64,
+
+ /// Number of tasks that were polled by the worker.
+ poll_count: u64,
+
+ /// Number of tasks polled when the worker entered park. This is used to
+ /// track the noop count.
+ poll_count_on_last_park: u64,
+
+ /// Number of tasks that were scheduled locally on this worker.
+ local_schedule_count: u64,
+
+ /// Number of tasks moved to the global queue to make space in the local
+ /// queue
+ overflow_count: u64,
+
+ /// The total busy duration in nanoseconds.
+ busy_duration_total: u64,
+ last_resume_time: Instant,
+}
+
+impl MetricsBatch {
+ pub(crate) fn new() -> MetricsBatch {
+ MetricsBatch {
+ park_count: 0,
+ noop_count: 0,
+ steal_count: 0,
+ steal_operations: 0,
+ poll_count: 0,
+ poll_count_on_last_park: 0,
+ local_schedule_count: 0,
+ overflow_count: 0,
+ busy_duration_total: 0,
+ last_resume_time: Instant::now(),
+ }
+ }
+
+ pub(crate) fn submit(&mut self, worker: &WorkerMetrics) {
+ worker.park_count.store(self.park_count, Relaxed);
+ worker.noop_count.store(self.noop_count, Relaxed);
+ worker.steal_count.store(self.steal_count, Relaxed);
+ worker
+ .steal_operations
+ .store(self.steal_operations, Relaxed);
+ worker.poll_count.store(self.poll_count, Relaxed);
+
+ worker
+ .busy_duration_total
+ .store(self.busy_duration_total, Relaxed);
+
+ worker
+ .local_schedule_count
+ .store(self.local_schedule_count, Relaxed);
+ worker.overflow_count.store(self.overflow_count, Relaxed);
+ }
+
+ /// The worker is about to park.
+ pub(crate) fn about_to_park(&mut self) {
+ self.park_count += 1;
+
+ if self.poll_count_on_last_park == self.poll_count {
+ self.noop_count += 1;
+ } else {
+ self.poll_count_on_last_park = self.poll_count;
+ }
+
+ let busy_duration = self.last_resume_time.elapsed();
+ let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX);
+ self.busy_duration_total += busy_duration;
+ }
+
+ pub(crate) fn returned_from_park(&mut self) {
+ self.last_resume_time = Instant::now();
+ }
+
+ pub(crate) fn inc_local_schedule_count(&mut self) {
+ self.local_schedule_count += 1;
+ }
+
+ pub(crate) fn incr_poll_count(&mut self) {
+ self.poll_count += 1;
+ }
+}
+
+cfg_rt_multi_thread! {
+ impl MetricsBatch {
+ pub(crate) fn incr_steal_count(&mut self, by: u16) {
+ self.steal_count += by as u64;
+ }
+
+ pub(crate) fn incr_steal_operations(&mut self) {
+ self.steal_operations += 1;
+ }
+
+ pub(crate) fn incr_overflow_count(&mut self) {
+ self.overflow_count += 1;
+ }
+ }
+}
diff --git a/src/runtime/metrics/io.rs b/src/runtime/metrics/io.rs
new file mode 100644
index 0000000..06efdd4
--- /dev/null
+++ b/src/runtime/metrics/io.rs
@@ -0,0 +1,24 @@
+#![cfg_attr(not(feature = "net"), allow(dead_code))]
+
+use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed};
+
+#[derive(Default)]
+pub(crate) struct IoDriverMetrics {
+ pub(super) fd_registered_count: AtomicU64,
+ pub(super) fd_deregistered_count: AtomicU64,
+ pub(super) ready_count: AtomicU64,
+}
+
+impl IoDriverMetrics {
+ pub(crate) fn incr_fd_count(&self) {
+ self.fd_registered_count.fetch_add(1, Relaxed);
+ }
+
+ pub(crate) fn dec_fd_count(&self) {
+ self.fd_deregistered_count.fetch_add(1, Relaxed);
+ }
+
+ pub(crate) fn incr_ready_count_by(&self, amt: u64) {
+ self.ready_count.fetch_add(amt, Relaxed);
+ }
+}
diff --git a/src/runtime/metrics/mock.rs b/src/runtime/metrics/mock.rs
new file mode 100644
index 0000000..c388dc0
--- /dev/null
+++ b/src/runtime/metrics/mock.rs
@@ -0,0 +1,44 @@
+//! This file contains mocks of the types in src/runtime/metrics
+
+pub(crate) struct SchedulerMetrics {}
+
+pub(crate) struct WorkerMetrics {}
+
+pub(crate) struct MetricsBatch {}
+
+impl SchedulerMetrics {
+ pub(crate) fn new() -> Self {
+ Self {}
+ }
+
+ /// Increment the number of tasks scheduled externally
+ pub(crate) fn inc_remote_schedule_count(&self) {}
+}
+
+impl WorkerMetrics {
+ pub(crate) fn new() -> Self {
+ Self {}
+ }
+
+ pub(crate) fn set_queue_depth(&self, _len: usize) {}
+}
+
+impl MetricsBatch {
+ pub(crate) fn new() -> Self {
+ Self {}
+ }
+
+ pub(crate) fn submit(&mut self, _to: &WorkerMetrics) {}
+ pub(crate) fn about_to_park(&mut self) {}
+ pub(crate) fn returned_from_park(&mut self) {}
+ pub(crate) fn incr_poll_count(&mut self) {}
+ pub(crate) fn inc_local_schedule_count(&mut self) {}
+}
+
+cfg_rt_multi_thread! {
+ impl MetricsBatch {
+ pub(crate) fn incr_steal_count(&mut self, _by: u16) {}
+ pub(crate) fn incr_steal_operations(&mut self) {}
+ pub(crate) fn incr_overflow_count(&mut self) {}
+ }
+}
diff --git a/src/runtime/metrics/mod.rs b/src/runtime/metrics/mod.rs
new file mode 100644
index 0000000..4b96f1b
--- /dev/null
+++ b/src/runtime/metrics/mod.rs
@@ -0,0 +1,35 @@
+//! This module contains information need to view information about how the
+//! runtime is performing.
+//!
+//! **Note**: This is an [unstable API][unstable]. The public API of types in
+//! this module may break in 1.x releases. See [the documentation on unstable
+//! features][unstable] for details.
+//!
+//! [unstable]: crate#unstable-features
+#![allow(clippy::module_inception)]
+
+cfg_metrics! {
+ mod batch;
+ pub(crate) use batch::MetricsBatch;
+
+ mod runtime;
+ #[allow(unreachable_pub)] // rust-lang/rust#57411
+ pub use runtime::RuntimeMetrics;
+
+ mod scheduler;
+ pub(crate) use scheduler::SchedulerMetrics;
+
+ mod worker;
+ pub(crate) use worker::WorkerMetrics;
+
+ cfg_net! {
+ mod io;
+ pub(crate) use io::IoDriverMetrics;
+ }
+}
+
+cfg_not_metrics! {
+ mod mock;
+
+ pub(crate) use mock::{SchedulerMetrics, WorkerMetrics, MetricsBatch};
+}
diff --git a/src/runtime/metrics/runtime.rs b/src/runtime/metrics/runtime.rs
new file mode 100644
index 0000000..d29cb3d
--- /dev/null
+++ b/src/runtime/metrics/runtime.rs
@@ -0,0 +1,658 @@
+use crate::runtime::Handle;
+
+use std::sync::atomic::Ordering::Relaxed;
+use std::time::Duration;
+
+/// Handle to the runtime's metrics.
+///
+/// This handle is internally reference-counted and can be freely cloned. A
+/// `RuntimeMetrics` handle is obtained using the [`Runtime::metrics`] method.
+///
+/// [`Runtime::metrics`]: crate::runtime::Runtime::metrics()
+#[derive(Clone, Debug)]
+pub struct RuntimeMetrics {
+ handle: Handle,
+}
+
+impl RuntimeMetrics {
+ pub(crate) fn new(handle: Handle) -> RuntimeMetrics {
+ RuntimeMetrics { handle }
+ }
+
+ /// Returns the number of worker threads used by the runtime.
+ ///
+ /// The number of workers is set by configuring `worker_threads` on
+ /// `runtime::Builder`. When using the `current_thread` runtime, the return
+ /// value is always `1`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.num_workers();
+ /// println!("Runtime is using {} workers", n);
+ /// }
+ /// ```
+ pub fn num_workers(&self) -> usize {
+ self.handle.inner.num_workers()
+ }
+
+ /// Returns the number of additional threads spawned by the runtime.
+ ///
+ /// The number of workers is set by configuring `max_blocking_threads` on
+ /// `runtime::Builder`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let _ = tokio::task::spawn_blocking(move || {
+ /// // Stand-in for compute-heavy work or using synchronous APIs
+ /// 1 + 1
+ /// }).await;
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.num_blocking_threads();
+ /// println!("Runtime has created {} threads", n);
+ /// }
+ /// ```
+ pub fn num_blocking_threads(&self) -> usize {
+ self.handle.inner.num_blocking_threads()
+ }
+
+ /// Returns the number of idle threads, which hve spawned by the runtime
+ /// for `spawn_blocking` calls.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let _ = tokio::task::spawn_blocking(move || {
+ /// // Stand-in for compute-heavy work or using synchronous APIs
+ /// 1 + 1
+ /// }).await;
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.num_idle_blocking_threads();
+ /// println!("Runtime has {} idle blocking thread pool threads", n);
+ /// }
+ /// ```
+ pub fn num_idle_blocking_threads(&self) -> usize {
+ self.handle.inner.num_idle_blocking_threads()
+ }
+
+ /// Returns the number of tasks scheduled from **outside** of the runtime.
+ ///
+ /// The remote schedule count starts at zero when the runtime is created and
+ /// increases by one each time a task is woken from **outside** of the
+ /// runtime. This usually means that a task is spawned or notified from a
+ /// non-runtime thread and must be queued using the Runtime's injection
+ /// queue, which tends to be slower.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.remote_schedule_count();
+ /// println!("{} tasks were scheduled from outside the runtime", n);
+ /// }
+ /// ```
+ pub fn remote_schedule_count(&self) -> u64 {
+ self.handle
+ .inner
+ .scheduler_metrics()
+ .remote_schedule_count
+ .load(Relaxed)
+ }
+
+ /// Returns the total number of times the given worker thread has parked.
+ ///
+ /// The worker park count starts at zero when the runtime is created and
+ /// increases by one each time the worker parks the thread waiting for new
+ /// inbound events to process. This usually means the worker has processed
+ /// all pending work and is currently idle.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_park_count(0);
+ /// println!("worker 0 parked {} times", n);
+ /// }
+ /// ```
+ pub fn worker_park_count(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .park_count
+ .load(Relaxed)
+ }
+
+ /// Returns the number of times the given worker thread unparked but
+ /// performed no work before parking again.
+ ///
+ /// The worker no-op count starts at zero when the runtime is created and
+ /// increases by one each time the worker unparks the thread but finds no
+ /// new work and goes back to sleep. This indicates a false-positive wake up.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_noop_count(0);
+ /// println!("worker 0 had {} no-op unparks", n);
+ /// }
+ /// ```
+ pub fn worker_noop_count(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .noop_count
+ .load(Relaxed)
+ }
+
+ /// Returns the number of tasks the given worker thread stole from
+ /// another worker thread.
+ ///
+ /// This metric only applies to the **multi-threaded** runtime and will
+ /// always return `0` when using the current thread runtime.
+ ///
+ /// The worker steal count starts at zero when the runtime is created and
+ /// increases by `N` each time the worker has processed its scheduled queue
+ /// and successfully steals `N` more pending tasks from another worker.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_steal_count(0);
+ /// println!("worker 0 has stolen {} tasks", n);
+ /// }
+ /// ```
+ pub fn worker_steal_count(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .steal_count
+ .load(Relaxed)
+ }
+
+ /// Returns the number of times the given worker thread stole tasks from
+ /// another worker thread.
+ ///
+ /// This metric only applies to the **multi-threaded** runtime and will
+ /// always return `0` when using the current thread runtime.
+ ///
+ /// The worker steal count starts at zero when the runtime is created and
+ /// increases by one each time the worker has processed its scheduled queue
+ /// and successfully steals more pending tasks from another worker.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_steal_operations(0);
+ /// println!("worker 0 has stolen tasks {} times", n);
+ /// }
+ /// ```
+ pub fn worker_steal_operations(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .steal_operations
+ .load(Relaxed)
+ }
+
+ /// Returns the number of tasks the given worker thread has polled.
+ ///
+ /// The worker poll count starts at zero when the runtime is created and
+ /// increases by one each time the worker polls a scheduled task.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_poll_count(0);
+ /// println!("worker 0 has polled {} tasks", n);
+ /// }
+ /// ```
+ pub fn worker_poll_count(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .poll_count
+ .load(Relaxed)
+ }
+
+ /// Returns the amount of time the given worker thread has been busy.
+ ///
+ /// The worker busy duration starts at zero when the runtime is created and
+ /// increases whenever the worker is spending time processing work. Using
+ /// this value can indicate the load of the given worker. If a lot of time
+ /// is spent busy, then the worker is under load and will check for inbound
+ /// events less often.
+ ///
+ /// The timer is monotonically increasing. It is never decremented or reset
+ /// to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_total_busy_duration(0);
+ /// println!("worker 0 was busy for a total of {:?}", n);
+ /// }
+ /// ```
+ pub fn worker_total_busy_duration(&self, worker: usize) -> Duration {
+ let nanos = self
+ .handle
+ .inner
+ .worker_metrics(worker)
+ .busy_duration_total
+ .load(Relaxed);
+ Duration::from_nanos(nanos)
+ }
+
+ /// Returns the number of tasks scheduled from **within** the runtime on the
+ /// given worker's local queue.
+ ///
+ /// The local schedule count starts at zero when the runtime is created and
+ /// increases by one each time a task is woken from **inside** of the
+ /// runtime on the given worker. This usually means that a task is spawned
+ /// or notified from within a runtime thread and will be queued on the
+ /// worker-local queue.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_local_schedule_count(0);
+ /// println!("{} tasks were scheduled on the worker's local queue", n);
+ /// }
+ /// ```
+ pub fn worker_local_schedule_count(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .local_schedule_count
+ .load(Relaxed)
+ }
+
+ /// Returns the number of times the given worker thread saturated its local
+ /// queue.
+ ///
+ /// This metric only applies to the **multi-threaded** scheduler.
+ ///
+ /// The worker steal count starts at zero when the runtime is created and
+ /// increases by one each time the worker attempts to schedule a task
+ /// locally, but its local queue is full. When this happens, half of the
+ /// local queue is moved to the injection queue.
+ ///
+ /// The counter is monotonically increasing. It is never decremented or
+ /// reset to zero.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_overflow_count(0);
+ /// println!("worker 0 has overflowed its queue {} times", n);
+ /// }
+ /// ```
+ pub fn worker_overflow_count(&self, worker: usize) -> u64 {
+ self.handle
+ .inner
+ .worker_metrics(worker)
+ .overflow_count
+ .load(Relaxed)
+ }
+
+ /// Returns the number of tasks currently scheduled in the runtime's
+ /// injection queue.
+ ///
+ /// Tasks that are spawned or notified from a non-runtime thread are
+ /// scheduled using the runtime's injection queue. This metric returns the
+ /// **current** number of tasks pending in the injection queue. As such, the
+ /// returned value may increase or decrease as new tasks are scheduled and
+ /// processed.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.injection_queue_depth();
+ /// println!("{} tasks currently pending in the runtime's injection queue", n);
+ /// }
+ /// ```
+ pub fn injection_queue_depth(&self) -> usize {
+ self.handle.inner.injection_queue_depth()
+ }
+
+ /// Returns the number of tasks currently scheduled in the given worker's
+ /// local queue.
+ ///
+ /// Tasks that are spawned or notified from within a runtime thread are
+ /// scheduled using that worker's local queue. This metric returns the
+ /// **current** number of tasks pending in the worker's local queue. As
+ /// such, the returned value may increase or decrease as new tasks are
+ /// scheduled and processed.
+ ///
+ /// # Arguments
+ ///
+ /// `worker` is the index of the worker being queried. The given value must
+ /// be between 0 and `num_workers()`. The index uniquely identifies a single
+ /// worker and will continue to identify the worker throughout the lifetime
+ /// of the runtime instance.
+ ///
+ /// # Panics
+ ///
+ /// The method panics when `worker` represents an invalid worker, i.e. is
+ /// greater than or equal to `num_workers()`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.worker_local_queue_depth(0);
+ /// println!("{} tasks currently pending in worker 0's local queue", n);
+ /// }
+ /// ```
+ pub fn worker_local_queue_depth(&self, worker: usize) -> usize {
+ self.handle.inner.worker_local_queue_depth(worker)
+ }
+
+ /// Returns the number of tasks currently scheduled in the blocking
+ /// thread pool, spawned using `spawn_blocking`.
+ ///
+ /// This metric returns the **current** number of tasks pending in
+ /// blocking thread pool. As such, the returned value may increase
+ /// or decrease as new tasks are scheduled and processed.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.blocking_queue_depth();
+ /// println!("{} tasks currently pending in the blocking thread pool", n);
+ /// }
+ /// ```
+ pub fn blocking_queue_depth(&self) -> usize {
+ self.handle.inner.blocking_queue_depth()
+ }
+}
+
+cfg_net! {
+ impl RuntimeMetrics {
+ /// Returns the number of file descriptors that have been registered with the
+ /// runtime's I/O driver.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let registered_fds = metrics.io_driver_fd_registered_count();
+ /// println!("{} fds have been registered with the runtime's I/O driver.", registered_fds);
+ ///
+ /// let deregistered_fds = metrics.io_driver_fd_deregistered_count();
+ ///
+ /// let current_fd_count = registered_fds - deregistered_fds;
+ /// println!("{} fds are currently registered by the runtime's I/O driver.", current_fd_count);
+ /// }
+ /// ```
+ pub fn io_driver_fd_registered_count(&self) -> u64 {
+ self.with_io_driver_metrics(|m| {
+ m.fd_registered_count.load(Relaxed)
+ })
+ }
+
+ /// Returns the number of file descriptors that have been deregistered by the
+ /// runtime's I/O driver.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.io_driver_fd_deregistered_count();
+ /// println!("{} fds have been deregistered by the runtime's I/O driver.", n);
+ /// }
+ /// ```
+ pub fn io_driver_fd_deregistered_count(&self) -> u64 {
+ self.with_io_driver_metrics(|m| {
+ m.fd_deregistered_count.load(Relaxed)
+ })
+ }
+
+ /// Returns the number of ready events processed by the runtime's
+ /// I/O driver.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Handle;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let metrics = Handle::current().metrics();
+ ///
+ /// let n = metrics.io_driver_ready_count();
+ /// println!("{} ready events processed by the runtime's I/O driver.", n);
+ /// }
+ /// ```
+ pub fn io_driver_ready_count(&self) -> u64 {
+ self.with_io_driver_metrics(|m| m.ready_count.load(Relaxed))
+ }
+
+ fn with_io_driver_metrics<F>(&self, f: F) -> u64
+ where
+ F: Fn(&super::IoDriverMetrics) -> u64,
+ {
+ // TODO: Investigate if this should return 0, most of our metrics always increase
+ // thus this breaks that guarantee.
+ self.handle
+ .inner
+ .driver()
+ .io
+ .as_ref()
+ .map(|h| f(&h.metrics))
+ .unwrap_or(0)
+ }
+ }
+}
diff --git a/src/runtime/metrics/scheduler.rs b/src/runtime/metrics/scheduler.rs
new file mode 100644
index 0000000..d1ba3b6
--- /dev/null
+++ b/src/runtime/metrics/scheduler.rs
@@ -0,0 +1,27 @@
+use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed};
+
+/// Retrieves metrics from the Tokio runtime.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [unstable]: crate#unstable-features
+#[derive(Debug)]
+pub(crate) struct SchedulerMetrics {
+ /// Number of tasks that are scheduled from outside the runtime.
+ pub(super) remote_schedule_count: AtomicU64,
+}
+
+impl SchedulerMetrics {
+ pub(crate) fn new() -> SchedulerMetrics {
+ SchedulerMetrics {
+ remote_schedule_count: AtomicU64::new(0),
+ }
+ }
+
+ /// Increment the number of tasks scheduled externally
+ pub(crate) fn inc_remote_schedule_count(&self) {
+ self.remote_schedule_count.fetch_add(1, Relaxed);
+ }
+}
diff --git a/src/runtime/metrics/worker.rs b/src/runtime/metrics/worker.rs
new file mode 100644
index 0000000..a40c76e
--- /dev/null
+++ b/src/runtime/metrics/worker.rs
@@ -0,0 +1,65 @@
+use crate::loom::sync::atomic::Ordering::Relaxed;
+use crate::loom::sync::atomic::{AtomicU64, AtomicUsize};
+
+/// Retrieve runtime worker metrics.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [unstable]: crate#unstable-features
+#[derive(Debug)]
+#[repr(align(128))]
+pub(crate) struct WorkerMetrics {
+ /// Number of times the worker parked.
+ pub(crate) park_count: AtomicU64,
+
+ /// Number of times the worker woke then parked again without doing work.
+ pub(crate) noop_count: AtomicU64,
+
+ /// Number of tasks the worker stole.
+ pub(crate) steal_count: AtomicU64,
+
+ /// Number of times the worker stole
+ pub(crate) steal_operations: AtomicU64,
+
+ /// Number of tasks the worker polled.
+ pub(crate) poll_count: AtomicU64,
+
+ /// Amount of time the worker spent doing work vs. parking.
+ pub(crate) busy_duration_total: AtomicU64,
+
+ /// Number of tasks scheduled for execution on the worker's local queue.
+ pub(crate) local_schedule_count: AtomicU64,
+
+ /// Number of tasks moved from the local queue to the global queue to free space.
+ pub(crate) overflow_count: AtomicU64,
+
+ /// Number of tasks currently in the local queue. Used only by the
+ /// current-thread scheduler.
+ pub(crate) queue_depth: AtomicUsize,
+}
+
+impl WorkerMetrics {
+ pub(crate) fn new() -> WorkerMetrics {
+ WorkerMetrics {
+ park_count: AtomicU64::new(0),
+ noop_count: AtomicU64::new(0),
+ steal_count: AtomicU64::new(0),
+ steal_operations: AtomicU64::new(0),
+ poll_count: AtomicU64::new(0),
+ overflow_count: AtomicU64::new(0),
+ busy_duration_total: AtomicU64::new(0),
+ local_schedule_count: AtomicU64::new(0),
+ queue_depth: AtomicUsize::new(0),
+ }
+ }
+
+ pub(crate) fn queue_depth(&self) -> usize {
+ self.queue_depth.load(Relaxed)
+ }
+
+ pub(crate) fn set_queue_depth(&self, len: usize) {
+ self.queue_depth.store(len, Relaxed);
+ }
+}
diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs
index 96bb47c..b6f43ea 100644
--- a/src/runtime/mod.rs
+++ b/src/runtime/mod.rs
@@ -137,7 +137,7 @@
//! use tokio::runtime;
//!
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
-//! let basic_rt = runtime::Builder::new_current_thread()
+//! let rt = runtime::Builder::new_current_thread()
//! .build()?;
//! # Ok(()) }
//! ```
@@ -166,7 +166,6 @@
//! [`tokio::main`]: ../attr.main.html
//! [runtime builder]: crate::runtime::Builder
//! [`Runtime::new`]: crate::runtime::Runtime::new
-//! [`Builder::basic_scheduler`]: crate::runtime::Builder::basic_scheduler
//! [`Builder::threaded_scheduler`]: crate::runtime::Builder::threaded_scheduler
//! [`Builder::enable_io`]: crate::runtime::Builder::enable_io
//! [`Builder::enable_time`]: crate::runtime::Builder::enable_time
@@ -174,427 +173,89 @@
// At the top due to macros
#[cfg(test)]
+#[cfg(not(tokio_wasm))]
#[macro_use]
mod tests;
-pub(crate) mod enter;
+pub(crate) mod context;
-pub(crate) mod task;
+pub(crate) mod coop;
-cfg_stats! {
- pub mod stats;
+pub(crate) mod park;
+
+mod driver;
+
+pub(crate) mod scheduler;
+
+cfg_io_driver_impl! {
+ pub(crate) mod io;
}
-cfg_not_stats! {
- pub(crate) mod stats;
+
+cfg_process_driver! {
+ mod process;
+}
+
+cfg_time! {
+ pub(crate) mod time;
+}
+
+cfg_signal_internal_and_unix! {
+ pub(crate) mod signal;
}
cfg_rt! {
- mod basic_scheduler;
- use basic_scheduler::BasicScheduler;
+ pub(crate) mod task;
+
+ mod config;
+ use config::Config;
mod blocking;
- use blocking::BlockingPool;
+ #[cfg_attr(tokio_wasi, allow(unused_imports))]
pub(crate) use blocking::spawn_blocking;
+ cfg_trace! {
+ pub(crate) use blocking::Mandatory;
+ }
+
+ cfg_fs! {
+ pub(crate) use blocking::spawn_mandatory_blocking;
+ }
+
mod builder;
pub use self::builder::Builder;
+ cfg_unstable! {
+ pub use self::builder::UnhandledPanic;
+ pub use crate::util::rand::RngSeed;
+ }
- pub(crate) mod context;
- pub(crate) mod driver;
-
- use self::enter::enter;
+ mod defer;
+ pub(crate) use defer::Defer;
mod handle;
pub use handle::{EnterGuard, Handle, TryCurrentError};
- mod spawner;
- use self::spawner::Spawner;
-}
+ mod runtime;
+ pub use runtime::{Runtime, RuntimeFlavor};
-cfg_rt_multi_thread! {
- mod park;
- use park::Parker;
-}
+ mod thread_id;
+ pub(crate) use thread_id::ThreadId;
-cfg_rt_multi_thread! {
- mod queue;
+ cfg_metrics! {
+ mod metrics;
+ pub use metrics::RuntimeMetrics;
- pub(crate) mod thread_pool;
- use self::thread_pool::ThreadPool;
-}
+ pub(crate) use metrics::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
-cfg_rt! {
- use crate::task::JoinHandle;
-
- use std::future::Future;
- use std::time::Duration;
-
- /// The Tokio runtime.
- ///
- /// The runtime provides an I/O driver, task scheduler, [timer], and
- /// blocking pool, necessary for running asynchronous tasks.
- ///
- /// Instances of `Runtime` can be created using [`new`], or [`Builder`].
- /// However, most users will use the `#[tokio::main]` annotation on their
- /// entry point instead.
- ///
- /// See [module level][mod] documentation for more details.
- ///
- /// # Shutdown
- ///
- /// Shutting down the runtime is done by dropping the value. The current
- /// thread will block until the shut down operation has completed.
- ///
- /// * Drain any scheduled work queues.
- /// * Drop any futures that have not yet completed.
- /// * Drop the reactor.
- ///
- /// Once the reactor has dropped, any outstanding I/O resources bound to
- /// that reactor will no longer function. Calling any method on them will
- /// result in an error.
- ///
- /// # Sharing
- ///
- /// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it
- /// in a `Arc`. Most fn take `&self` to allow you to call them concurrently
- /// across multiple threads.
- ///
- /// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of
- /// the runtime type and this can be achieved via `Arc::try_unwrap` when only
- /// one strong count reference is left over.
- ///
- /// [timer]: crate::time
- /// [mod]: index.html
- /// [`new`]: method@Self::new
- /// [`Builder`]: struct@Builder
- #[derive(Debug)]
- pub struct Runtime {
- /// Task executor
- kind: Kind,
-
- /// Handle to runtime, also contains driver handles
- handle: Handle,
-
- /// Blocking pool handle, used to signal shutdown
- blocking_pool: BlockingPool,
+ cfg_net! {
+ pub(crate) use metrics::IoDriverMetrics;
+ }
}
- /// The runtime executor is either a thread-pool or a current-thread executor.
- #[derive(Debug)]
- enum Kind {
- /// Execute all tasks on the current-thread.
- CurrentThread(BasicScheduler<driver::Driver>),
-
- /// Execute tasks across multiple threads.
- #[cfg(feature = "rt-multi-thread")]
- ThreadPool(ThreadPool),
+ cfg_not_metrics! {
+ pub(crate) mod metrics;
+ pub(crate) use metrics::{SchedulerMetrics, WorkerMetrics, MetricsBatch};
}
/// After thread starts / before thread stops
type Callback = std::sync::Arc<dyn Fn() + Send + Sync>;
-
- impl Runtime {
- /// Creates a new runtime instance with default configuration values.
- ///
- /// This results in the multi threaded scheduler, I/O driver, and time driver being
- /// initialized.
- ///
- /// Most applications will not need to call this function directly. Instead,
- /// they will use the [`#[tokio::main]` attribute][main]. When a more complex
- /// configuration is necessary, the [runtime builder] may be used.
- ///
- /// See [module level][mod] documentation for more details.
- ///
- /// # Examples
- ///
- /// Creating a new `Runtime` with default configuration values.
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- ///
- /// let rt = Runtime::new()
- /// .unwrap();
- ///
- /// // Use the runtime...
- /// ```
- ///
- /// [mod]: index.html
- /// [main]: ../attr.main.html
- /// [threaded scheduler]: index.html#threaded-scheduler
- /// [basic scheduler]: index.html#basic-scheduler
- /// [runtime builder]: crate::runtime::Builder
- #[cfg(feature = "rt-multi-thread")]
- #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
- pub fn new() -> std::io::Result<Runtime> {
- Builder::new_multi_thread().enable_all().build()
- }
-
- /// Returns a handle to the runtime's spawner.
- ///
- /// The returned handle can be used to spawn tasks that run on this runtime, and can
- /// be cloned to allow moving the `Handle` to other threads.
- ///
- /// # Examples
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- ///
- /// let rt = Runtime::new()
- /// .unwrap();
- ///
- /// let handle = rt.handle();
- ///
- /// // Use the handle...
- /// ```
- pub fn handle(&self) -> &Handle {
- &self.handle
- }
-
- /// Spawns a future onto the Tokio runtime.
- ///
- /// This spawns the given future onto the runtime's executor, usually a
- /// thread pool. The thread pool is then responsible for polling the future
- /// until it completes.
- ///
- /// See [module level][mod] documentation for more details.
- ///
- /// [mod]: index.html
- ///
- /// # Examples
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- ///
- /// # fn dox() {
- /// // Create the runtime
- /// let rt = Runtime::new().unwrap();
- ///
- /// // Spawn a future onto the runtime
- /// rt.spawn(async {
- /// println!("now running on a worker thread");
- /// });
- /// # }
- /// ```
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
- where
- F: Future + Send + 'static,
- F::Output: Send + 'static,
- {
- self.handle.spawn(future)
- }
-
- /// Runs the provided function on an executor dedicated to blocking operations.
- ///
- /// # Examples
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- ///
- /// # fn dox() {
- /// // Create the runtime
- /// let rt = Runtime::new().unwrap();
- ///
- /// // Spawn a blocking function onto the runtime
- /// rt.spawn_blocking(|| {
- /// println!("now running on a worker thread");
- /// });
- /// # }
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
- where
- F: FnOnce() -> R + Send + 'static,
- R: Send + 'static,
- {
- self.handle.spawn_blocking(func)
- }
-
- /// Runs a future to completion on the Tokio runtime. This is the
- /// runtime's entry point.
- ///
- /// This runs the given future on the current thread, blocking until it is
- /// complete, and yielding its resolved result. Any tasks or timers
- /// which the future spawns internally will be executed on the runtime.
- ///
- /// # Multi thread scheduler
- ///
- /// When the multi thread scheduler is used this will allow futures
- /// to run within the io driver and timer context of the overall runtime.
- ///
- /// # Current thread scheduler
- ///
- /// When the current thread scheduler is enabled `block_on`
- /// can be called concurrently from multiple threads. The first call
- /// will take ownership of the io and timer drivers. This means
- /// other threads which do not own the drivers will hook into that one.
- /// When the first `block_on` completes, other threads will be able to
- /// "steal" the driver to allow continued execution of their futures.
- ///
- /// # Panics
- ///
- /// This function panics if the provided future panics, or if called within an
- /// asynchronous execution context.
- ///
- /// # Examples
- ///
- /// ```no_run
- /// use tokio::runtime::Runtime;
- ///
- /// // Create the runtime
- /// let rt = Runtime::new().unwrap();
- ///
- /// // Execute the future, blocking the current thread until completion
- /// rt.block_on(async {
- /// println!("hello");
- /// });
- /// ```
- ///
- /// [handle]: fn@Handle::block_on
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub fn block_on<F: Future>(&self, future: F) -> F::Output {
- #[cfg(all(tokio_unstable, feature = "tracing"))]
- let future = crate::util::trace::task(future, "block_on", None);
-
- let _enter = self.enter();
-
- match &self.kind {
- Kind::CurrentThread(exec) => exec.block_on(future),
- #[cfg(feature = "rt-multi-thread")]
- Kind::ThreadPool(exec) => exec.block_on(future),
- }
- }
-
- /// Enters the runtime context.
- ///
- /// This allows you to construct types that must have an executor
- /// available on creation such as [`Sleep`] or [`TcpStream`]. It will
- /// also allow you to call methods such as [`tokio::spawn`].
- ///
- /// [`Sleep`]: struct@crate::time::Sleep
- /// [`TcpStream`]: struct@crate::net::TcpStream
- /// [`tokio::spawn`]: fn@crate::spawn
- ///
- /// # Example
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- ///
- /// fn function_that_spawns(msg: String) {
- /// // Had we not used `rt.enter` below, this would panic.
- /// tokio::spawn(async move {
- /// println!("{}", msg);
- /// });
- /// }
- ///
- /// fn main() {
- /// let rt = Runtime::new().unwrap();
- ///
- /// let s = "Hello World!".to_string();
- ///
- /// // By entering the context, we tie `tokio::spawn` to this executor.
- /// let _guard = rt.enter();
- /// function_that_spawns(s);
- /// }
- /// ```
- pub fn enter(&self) -> EnterGuard<'_> {
- self.handle.enter()
- }
-
- /// Shuts down the runtime, waiting for at most `duration` for all spawned
- /// task to shutdown.
- ///
- /// Usually, dropping a `Runtime` handle is sufficient as tasks are able to
- /// shutdown in a timely fashion. However, dropping a `Runtime` will wait
- /// indefinitely for all tasks to terminate, and there are cases where a long
- /// blocking task has been spawned, which can block dropping `Runtime`.
- ///
- /// In this case, calling `shutdown_timeout` with an explicit wait timeout
- /// can work. The `shutdown_timeout` will signal all tasks to shutdown and
- /// will wait for at most `duration` for all spawned tasks to terminate. If
- /// `timeout` elapses before all tasks are dropped, the function returns and
- /// outstanding tasks are potentially leaked.
- ///
- /// # Examples
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- /// use tokio::task;
- ///
- /// use std::thread;
- /// use std::time::Duration;
- ///
- /// fn main() {
- /// let runtime = Runtime::new().unwrap();
- ///
- /// runtime.block_on(async move {
- /// task::spawn_blocking(move || {
- /// thread::sleep(Duration::from_secs(10_000));
- /// });
- /// });
- ///
- /// runtime.shutdown_timeout(Duration::from_millis(100));
- /// }
- /// ```
- pub fn shutdown_timeout(mut self, duration: Duration) {
- // Wakeup and shutdown all the worker threads
- self.handle.clone().shutdown();
- self.blocking_pool.shutdown(Some(duration));
- }
-
- /// Shuts down the runtime, without waiting for any spawned tasks to shutdown.
- ///
- /// This can be useful if you want to drop a runtime from within another runtime.
- /// Normally, dropping a runtime will block indefinitely for spawned blocking tasks
- /// to complete, which would normally not be permitted within an asynchronous context.
- /// By calling `shutdown_background()`, you can drop the runtime from such a context.
- ///
- /// Note however, that because we do not wait for any blocking tasks to complete, this
- /// may result in a resource leak (in that any blocking tasks are still running until they
- /// return.
- ///
- /// This function is equivalent to calling `shutdown_timeout(Duration::of_nanos(0))`.
- ///
- /// ```
- /// use tokio::runtime::Runtime;
- ///
- /// fn main() {
- /// let runtime = Runtime::new().unwrap();
- ///
- /// runtime.block_on(async move {
- /// let inner_runtime = Runtime::new().unwrap();
- /// // ...
- /// inner_runtime.shutdown_background();
- /// });
- /// }
- /// ```
- pub fn shutdown_background(self) {
- self.shutdown_timeout(Duration::from_nanos(0))
- }
- }
-
- #[allow(clippy::single_match)] // there are comments in the error branch, so we don't want if-let
- impl Drop for Runtime {
- fn drop(&mut self) {
- match &mut self.kind {
- Kind::CurrentThread(basic) => {
- // This ensures that tasks spawned on the basic runtime are dropped inside the
- // runtime's context.
- match self::context::try_enter(self.handle.clone()) {
- Some(guard) => basic.set_context_guard(guard),
- None => {
- // The context thread-local has alread been destroyed.
- //
- // We don't set the guard in this case. Calls to tokio::spawn in task
- // destructors would fail regardless if this happens.
- },
- }
- },
- #[cfg(feature = "rt-multi-thread")]
- Kind::ThreadPool(_) => {
- // The threaded scheduler drops its tasks on its worker threads, which is
- // already in the runtime's context.
- },
- }
- }
- }
}
diff --git a/src/runtime/park.rs b/src/runtime/park.rs
index 033b9f2..dc86f42 100644
--- a/src/runtime/park.rs
+++ b/src/runtime/park.rs
@@ -1,153 +1,102 @@
-//! Parks the runtime.
-//!
-//! A combination of the various resource driver park handles.
+#![cfg_attr(not(feature = "full"), allow(dead_code))]
use crate::loom::sync::atomic::AtomicUsize;
use crate::loom::sync::{Arc, Condvar, Mutex};
-use crate::loom::thread;
-use crate::park::{Park, Unpark};
-use crate::runtime::driver::Driver;
-use crate::util::TryLock;
use std::sync::atomic::Ordering::SeqCst;
use std::time::Duration;
-pub(crate) struct Parker {
+#[derive(Debug)]
+pub(crate) struct ParkThread {
inner: Arc<Inner>,
}
-pub(crate) struct Unparker {
+/// Unblocks a thread that was blocked by `ParkThread`.
+#[derive(Clone, Debug)]
+pub(crate) struct UnparkThread {
inner: Arc<Inner>,
}
+#[derive(Debug)]
struct Inner {
- /// Avoids entering the park if possible
state: AtomicUsize,
-
- /// Used to coordinate access to the driver / condvar
mutex: Mutex<()>,
-
- /// Condvar to block on if the driver is unavailable.
condvar: Condvar,
-
- /// Resource (I/O, time, ...) driver
- shared: Arc<Shared>,
}
const EMPTY: usize = 0;
-const PARKED_CONDVAR: usize = 1;
-const PARKED_DRIVER: usize = 2;
-const NOTIFIED: usize = 3;
+const PARKED: usize = 1;
+const NOTIFIED: usize = 2;
-/// Shared across multiple Parker handles
-struct Shared {
- /// Shared driver. Only one thread at a time can use this
- driver: TryLock<Driver>,
-
- /// Unpark handle
- handle: <Driver as Park>::Unpark,
+tokio_thread_local! {
+ static CURRENT_PARKER: ParkThread = ParkThread::new();
}
-impl Parker {
- pub(crate) fn new(driver: Driver) -> Parker {
- let handle = driver.unpark();
+// Bit of a hack, but it is only for loom
+#[cfg(loom)]
+tokio_thread_local! {
+ static CURRENT_THREAD_PARK_COUNT: AtomicUsize = AtomicUsize::new(0);
+}
- Parker {
+// ==== impl ParkThread ====
+
+impl ParkThread {
+ pub(crate) fn new() -> Self {
+ Self {
inner: Arc::new(Inner {
state: AtomicUsize::new(EMPTY),
mutex: Mutex::new(()),
condvar: Condvar::new(),
- shared: Arc::new(Shared {
- driver: TryLock::new(driver),
- handle,
- }),
}),
}
}
-}
-impl Clone for Parker {
- fn clone(&self) -> Parker {
- Parker {
- inner: Arc::new(Inner {
- state: AtomicUsize::new(EMPTY),
- mutex: Mutex::new(()),
- condvar: Condvar::new(),
- shared: self.inner.shared.clone(),
- }),
- }
- }
-}
-
-impl Park for Parker {
- type Unpark = Unparker;
- type Error = ();
-
- fn unpark(&self) -> Unparker {
- Unparker {
- inner: self.inner.clone(),
- }
+ pub(crate) fn unpark(&self) -> UnparkThread {
+ let inner = self.inner.clone();
+ UnparkThread { inner }
}
- fn park(&mut self) -> Result<(), Self::Error> {
+ pub(crate) fn park(&mut self) {
+ #[cfg(loom)]
+ CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
self.inner.park();
- Ok(())
}
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- // Only parking with zero is supported...
- assert_eq!(duration, Duration::from_millis(0));
+ pub(crate) fn park_timeout(&mut self, duration: Duration) {
+ #[cfg(loom)]
+ CURRENT_THREAD_PARK_COUNT.with(|count| count.fetch_add(1, SeqCst));
- if let Some(mut driver) = self.inner.shared.driver.try_lock() {
- driver.park_timeout(duration).map_err(|_| ())
- } else {
- Ok(())
- }
+ // Wasm doesn't have threads, so just sleep.
+ #[cfg(not(tokio_wasm))]
+ self.inner.park_timeout(duration);
+ #[cfg(tokio_wasm)]
+ std::thread::sleep(duration);
}
- fn shutdown(&mut self) {
+ pub(crate) fn shutdown(&mut self) {
self.inner.shutdown();
}
}
-impl Unpark for Unparker {
- fn unpark(&self) {
- self.inner.unpark();
- }
-}
+// ==== impl Inner ====
impl Inner {
/// Parks the current thread for at most `dur`.
fn park(&self) {
- for _ in 0..3 {
- // If we were previously notified then we consume this notification and
- // return quickly.
- if self
- .state
- .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
- .is_ok()
- {
- return;
- }
-
- thread::yield_now();
+ // If we were previously notified then we consume this notification and
+ // return quickly.
+ if self
+ .state
+ .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
+ .is_ok()
+ {
+ return;
}
- if let Some(mut driver) = self.shared.driver.try_lock() {
- self.park_driver(&mut driver);
- } else {
- self.park_condvar();
- }
- }
-
- fn park_condvar(&self) {
// Otherwise we need to coordinate going to sleep
let mut m = self.mutex.lock();
- match self
- .state
- .compare_exchange(EMPTY, PARKED_CONDVAR, SeqCst, SeqCst)
- {
+ match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) {
Ok(_) => {}
Err(NOTIFIED) => {
// We must read here, even though we know it will be `NOTIFIED`.
@@ -180,33 +129,44 @@
}
}
- fn park_driver(&self, driver: &mut Driver) {
- match self
+ fn park_timeout(&self, dur: Duration) {
+ // Like `park` above we have a fast path for an already-notified thread,
+ // and afterwards we start coordinating for a sleep. Return quickly.
+ if self
.state
- .compare_exchange(EMPTY, PARKED_DRIVER, SeqCst, SeqCst)
+ .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
+ .is_ok()
{
+ return;
+ }
+
+ if dur == Duration::from_millis(0) {
+ return;
+ }
+
+ let m = self.mutex.lock();
+
+ match self.state.compare_exchange(EMPTY, PARKED, SeqCst, SeqCst) {
Ok(_) => {}
Err(NOTIFIED) => {
- // We must read here, even though we know it will be `NOTIFIED`.
- // This is because `unpark` may have been called again since we read
- // `NOTIFIED` in the `compare_exchange` above. We must perform an
- // acquire operation that synchronizes with that `unpark` to observe
- // any writes it made before the call to unpark. To do that we must
- // read from the write it made to `state`.
+ // We must read again here, see `park`.
let old = self.state.swap(EMPTY, SeqCst);
debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly");
return;
}
- Err(actual) => panic!("inconsistent park state; actual = {}", actual),
+ Err(actual) => panic!("inconsistent park_timeout state; actual = {}", actual),
}
- // TODO: don't unwrap
- driver.park().unwrap();
+ // Wait with a timeout, and if we spuriously wake up or otherwise wake up
+ // from a notification, we just want to unconditionally set the state back to
+ // empty, either consuming a notification or un-flagging ourselves as
+ // parked.
+ let (_m, _result) = self.condvar.wait_timeout(m, dur).unwrap();
match self.state.swap(EMPTY, SeqCst) {
- NOTIFIED => {} // got a notification, hurray!
- PARKED_DRIVER => {} // no notification, alas
+ NOTIFIED => {} // got a notification, hurray!
+ PARKED => {} // no notification, alas
n => panic!("inconsistent park_timeout state: {}", n),
}
}
@@ -218,15 +178,12 @@
// is already `NOTIFIED`. That is why this must be a swap rather than a
// compare-and-swap that returns if it reads `NOTIFIED` on failure.
match self.state.swap(NOTIFIED, SeqCst) {
- EMPTY => {} // no one was waiting
- NOTIFIED => {} // already unparked
- PARKED_CONDVAR => self.unpark_condvar(),
- PARKED_DRIVER => self.unpark_driver(),
- actual => panic!("inconsistent state in unpark; actual = {}", actual),
+ EMPTY => return, // no one was waiting
+ NOTIFIED => return, // already unparked
+ PARKED => {} // gotta go wake someone up
+ _ => panic!("inconsistent state in unpark"),
}
- }
- fn unpark_condvar(&self) {
// There is a period between when the parked thread sets `state` to
// `PARKED` (or last checked `state` in the case of a spurious wake
// up) and when it actually waits on `cvar`. If we were to notify
@@ -243,15 +200,154 @@
self.condvar.notify_one()
}
- fn unpark_driver(&self) {
- self.shared.handle.unpark();
- }
-
fn shutdown(&self) {
- if let Some(mut driver) = self.shared.driver.try_lock() {
- driver.shutdown();
- }
-
self.condvar.notify_all();
}
}
+
+impl Default for ParkThread {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+// ===== impl UnparkThread =====
+
+impl UnparkThread {
+ pub(crate) fn unpark(&self) {
+ self.inner.unpark();
+ }
+}
+
+use crate::loom::thread::AccessError;
+use std::future::Future;
+use std::marker::PhantomData;
+use std::mem;
+use std::rc::Rc;
+use std::task::{RawWaker, RawWakerVTable, Waker};
+
+/// Blocks the current thread using a condition variable.
+#[derive(Debug)]
+pub(crate) struct CachedParkThread {
+ _anchor: PhantomData<Rc<()>>,
+}
+
+impl CachedParkThread {
+ /// Creates a new `ParkThread` handle for the current thread.
+ ///
+ /// This type cannot be moved to other threads, so it should be created on
+ /// the thread that the caller intends to park.
+ pub(crate) fn new() -> CachedParkThread {
+ CachedParkThread {
+ _anchor: PhantomData,
+ }
+ }
+
+ pub(crate) fn waker(&self) -> Result<Waker, AccessError> {
+ self.unpark().map(|unpark| unpark.into_waker())
+ }
+
+ fn unpark(&self) -> Result<UnparkThread, AccessError> {
+ self.with_current(|park_thread| park_thread.unpark())
+ }
+
+ pub(crate) fn park(&mut self) {
+ self.with_current(|park_thread| park_thread.inner.park())
+ .unwrap();
+ }
+
+ pub(crate) fn park_timeout(&mut self, duration: Duration) {
+ self.with_current(|park_thread| park_thread.inner.park_timeout(duration))
+ .unwrap();
+ }
+
+ /// Gets a reference to the `ParkThread` handle for this thread.
+ fn with_current<F, R>(&self, f: F) -> Result<R, AccessError>
+ where
+ F: FnOnce(&ParkThread) -> R,
+ {
+ CURRENT_PARKER.try_with(|inner| f(inner))
+ }
+
+ pub(crate) fn block_on<F: Future>(&mut self, f: F) -> Result<F::Output, AccessError> {
+ use std::task::Context;
+ use std::task::Poll::Ready;
+
+ // `get_unpark()` should not return a Result
+ let waker = self.waker()?;
+ let mut cx = Context::from_waker(&waker);
+
+ pin!(f);
+
+ loop {
+ if let Ready(v) = crate::runtime::coop::budget(|| f.as_mut().poll(&mut cx)) {
+ return Ok(v);
+ }
+
+ // Wake any yielded tasks before parking in order to avoid
+ // blocking.
+ #[cfg(feature = "rt")]
+ crate::runtime::context::with_defer(|defer| defer.wake());
+
+ self.park();
+ }
+ }
+}
+
+impl UnparkThread {
+ pub(crate) fn into_waker(self) -> Waker {
+ unsafe {
+ let raw = unparker_to_raw_waker(self.inner);
+ Waker::from_raw(raw)
+ }
+ }
+}
+
+impl Inner {
+ #[allow(clippy::wrong_self_convention)]
+ fn into_raw(this: Arc<Inner>) -> *const () {
+ Arc::into_raw(this) as *const ()
+ }
+
+ unsafe fn from_raw(ptr: *const ()) -> Arc<Inner> {
+ Arc::from_raw(ptr as *const Inner)
+ }
+}
+
+unsafe fn unparker_to_raw_waker(unparker: Arc<Inner>) -> RawWaker {
+ RawWaker::new(
+ Inner::into_raw(unparker),
+ &RawWakerVTable::new(clone, wake, wake_by_ref, drop_waker),
+ )
+}
+
+unsafe fn clone(raw: *const ()) -> RawWaker {
+ let unparker = Inner::from_raw(raw);
+
+ // Increment the ref count
+ mem::forget(unparker.clone());
+
+ unparker_to_raw_waker(unparker)
+}
+
+unsafe fn drop_waker(raw: *const ()) {
+ let _ = Inner::from_raw(raw);
+}
+
+unsafe fn wake(raw: *const ()) {
+ let unparker = Inner::from_raw(raw);
+ unparker.unpark();
+}
+
+unsafe fn wake_by_ref(raw: *const ()) {
+ let unparker = Inner::from_raw(raw);
+ unparker.unpark();
+
+ // We don't actually own a reference to the unparker
+ mem::forget(unparker);
+}
+
+#[cfg(loom)]
+pub(crate) fn current_thread_park_count() -> usize {
+ CURRENT_THREAD_PARK_COUNT.with(|count| count.load(SeqCst))
+}
diff --git a/src/runtime/process.rs b/src/runtime/process.rs
new file mode 100644
index 0000000..df339b0
--- /dev/null
+++ b/src/runtime/process.rs
@@ -0,0 +1,44 @@
+#![cfg_attr(not(feature = "rt"), allow(dead_code))]
+
+//! Process driver.
+
+use crate::process::unix::GlobalOrphanQueue;
+use crate::runtime::driver;
+use crate::runtime::signal::{Driver as SignalDriver, Handle as SignalHandle};
+
+use std::time::Duration;
+
+/// Responsible for cleaning up orphaned child processes on Unix platforms.
+#[derive(Debug)]
+pub(crate) struct Driver {
+ park: SignalDriver,
+ signal_handle: SignalHandle,
+}
+
+// ===== impl Driver =====
+
+impl Driver {
+ /// Creates a new signal `Driver` instance that delegates wakeups to `park`.
+ pub(crate) fn new(park: SignalDriver) -> Self {
+ let signal_handle = park.handle();
+
+ Self {
+ park,
+ signal_handle,
+ }
+ }
+
+ pub(crate) fn park(&mut self, handle: &driver::Handle) {
+ self.park.park(handle);
+ GlobalOrphanQueue::reap_orphans(&self.signal_handle);
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) {
+ self.park.park_timeout(handle, duration);
+ GlobalOrphanQueue::reap_orphans(&self.signal_handle);
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &driver::Handle) {
+ self.park.shutdown(handle)
+ }
+}
diff --git a/src/runtime/runtime.rs b/src/runtime/runtime.rs
new file mode 100644
index 0000000..1985673
--- /dev/null
+++ b/src/runtime/runtime.rs
@@ -0,0 +1,423 @@
+use crate::runtime::blocking::BlockingPool;
+use crate::runtime::scheduler::CurrentThread;
+use crate::runtime::{context, EnterGuard, Handle};
+use crate::task::JoinHandle;
+
+use std::future::Future;
+use std::time::Duration;
+
+cfg_rt_multi_thread! {
+ use crate::runtime::Builder;
+ use crate::runtime::scheduler::MultiThread;
+}
+
+/// The Tokio runtime.
+///
+/// The runtime provides an I/O driver, task scheduler, [timer], and
+/// blocking pool, necessary for running asynchronous tasks.
+///
+/// Instances of `Runtime` can be created using [`new`], or [`Builder`].
+/// However, most users will use the `#[tokio::main]` annotation on their
+/// entry point instead.
+///
+/// See [module level][mod] documentation for more details.
+///
+/// # Shutdown
+///
+/// Shutting down the runtime is done by dropping the value. The current
+/// thread will block until the shut down operation has completed.
+///
+/// * Drain any scheduled work queues.
+/// * Drop any futures that have not yet completed.
+/// * Drop the reactor.
+///
+/// Once the reactor has dropped, any outstanding I/O resources bound to
+/// that reactor will no longer function. Calling any method on them will
+/// result in an error.
+///
+/// # Sharing
+///
+/// The Tokio runtime implements `Sync` and `Send` to allow you to wrap it
+/// in a `Arc`. Most fn take `&self` to allow you to call them concurrently
+/// across multiple threads.
+///
+/// Calls to `shutdown` and `shutdown_timeout` require exclusive ownership of
+/// the runtime type and this can be achieved via `Arc::try_unwrap` when only
+/// one strong count reference is left over.
+///
+/// [timer]: crate::time
+/// [mod]: index.html
+/// [`new`]: method@Self::new
+/// [`Builder`]: struct@Builder
+#[derive(Debug)]
+pub struct Runtime {
+ /// Task scheduler
+ scheduler: Scheduler,
+
+ /// Handle to runtime, also contains driver handles
+ handle: Handle,
+
+ /// Blocking pool handle, used to signal shutdown
+ blocking_pool: BlockingPool,
+}
+
+/// The flavor of a `Runtime`.
+///
+/// This is the return type for [`Handle::runtime_flavor`](crate::runtime::Handle::runtime_flavor()).
+#[derive(Debug, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum RuntimeFlavor {
+ /// The flavor that executes all tasks on the current thread.
+ CurrentThread,
+ /// The flavor that executes tasks across multiple threads.
+ MultiThread,
+}
+
+/// The runtime scheduler is either a multi-thread or a current-thread executor.
+#[derive(Debug)]
+pub(super) enum Scheduler {
+ /// Execute all tasks on the current-thread.
+ CurrentThread(CurrentThread),
+
+ /// Execute tasks across multiple threads.
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ MultiThread(MultiThread),
+}
+
+impl Runtime {
+ pub(super) fn from_parts(
+ scheduler: Scheduler,
+ handle: Handle,
+ blocking_pool: BlockingPool,
+ ) -> Runtime {
+ Runtime {
+ scheduler,
+ handle,
+ blocking_pool,
+ }
+ }
+
+ cfg_not_wasi! {
+ /// Creates a new runtime instance with default configuration values.
+ ///
+ /// This results in the multi threaded scheduler, I/O driver, and time driver being
+ /// initialized.
+ ///
+ /// Most applications will not need to call this function directly. Instead,
+ /// they will use the [`#[tokio::main]` attribute][main]. When a more complex
+ /// configuration is necessary, the [runtime builder] may be used.
+ ///
+ /// See [module level][mod] documentation for more details.
+ ///
+ /// # Examples
+ ///
+ /// Creating a new `Runtime` with default configuration values.
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ ///
+ /// let rt = Runtime::new()
+ /// .unwrap();
+ ///
+ /// // Use the runtime...
+ /// ```
+ ///
+ /// [mod]: index.html
+ /// [main]: ../attr.main.html
+ /// [threaded scheduler]: index.html#threaded-scheduler
+ /// [runtime builder]: crate::runtime::Builder
+ #[cfg(feature = "rt-multi-thread")]
+ #[cfg_attr(docsrs, doc(cfg(feature = "rt-multi-thread")))]
+ pub fn new() -> std::io::Result<Runtime> {
+ Builder::new_multi_thread().enable_all().build()
+ }
+ }
+
+ /// Returns a handle to the runtime's spawner.
+ ///
+ /// The returned handle can be used to spawn tasks that run on this runtime, and can
+ /// be cloned to allow moving the `Handle` to other threads.
+ ///
+ /// Calling [`Handle::block_on`] on a handle to a `current_thread` runtime is error-prone.
+ /// Refer to the documentation of [`Handle::block_on`] for more.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ ///
+ /// let rt = Runtime::new()
+ /// .unwrap();
+ ///
+ /// let handle = rt.handle();
+ ///
+ /// // Use the handle...
+ /// ```
+ pub fn handle(&self) -> &Handle {
+ &self.handle
+ }
+
+ /// Spawns a future onto the Tokio runtime.
+ ///
+ /// This spawns the given future onto the runtime's executor, usually a
+ /// thread pool. The thread pool is then responsible for polling the future
+ /// until it completes.
+ ///
+ /// The provided future will start running in the background immediately
+ /// when `spawn` is called, even if you don't await the returned
+ /// `JoinHandle`.
+ ///
+ /// See [module level][mod] documentation for more details.
+ ///
+ /// [mod]: index.html
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ ///
+ /// # fn dox() {
+ /// // Create the runtime
+ /// let rt = Runtime::new().unwrap();
+ ///
+ /// // Spawn a future onto the runtime
+ /// rt.spawn(async {
+ /// println!("now running on a worker thread");
+ /// });
+ /// # }
+ /// ```
+ #[track_caller]
+ pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ self.handle.spawn(future)
+ }
+
+ /// Runs the provided function on an executor dedicated to blocking operations.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ ///
+ /// # fn dox() {
+ /// // Create the runtime
+ /// let rt = Runtime::new().unwrap();
+ ///
+ /// // Spawn a blocking function onto the runtime
+ /// rt.spawn_blocking(|| {
+ /// println!("now running on a worker thread");
+ /// });
+ /// # }
+ #[track_caller]
+ pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+ where
+ F: FnOnce() -> R + Send + 'static,
+ R: Send + 'static,
+ {
+ self.handle.spawn_blocking(func)
+ }
+
+ /// Runs a future to completion on the Tokio runtime. This is the
+ /// runtime's entry point.
+ ///
+ /// This runs the given future on the current thread, blocking until it is
+ /// complete, and yielding its resolved result. Any tasks or timers
+ /// which the future spawns internally will be executed on the runtime.
+ ///
+ /// # Multi thread scheduler
+ ///
+ /// When the multi thread scheduler is used this will allow futures
+ /// to run within the io driver and timer context of the overall runtime.
+ ///
+ /// Any spawned tasks will continue running after `block_on` returns.
+ ///
+ /// # Current thread scheduler
+ ///
+ /// When the current thread scheduler is enabled `block_on`
+ /// can be called concurrently from multiple threads. The first call
+ /// will take ownership of the io and timer drivers. This means
+ /// other threads which do not own the drivers will hook into that one.
+ /// When the first `block_on` completes, other threads will be able to
+ /// "steal" the driver to allow continued execution of their futures.
+ ///
+ /// Any spawned tasks will be suspended after `block_on` returns. Calling
+ /// `block_on` again will resume previously spawned tasks.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if the provided future panics, or if called within an
+ /// asynchronous execution context.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use tokio::runtime::Runtime;
+ ///
+ /// // Create the runtime
+ /// let rt = Runtime::new().unwrap();
+ ///
+ /// // Execute the future, blocking the current thread until completion
+ /// rt.block_on(async {
+ /// println!("hello");
+ /// });
+ /// ```
+ ///
+ /// [handle]: fn@Handle::block_on
+ #[track_caller]
+ pub fn block_on<F: Future>(&self, future: F) -> F::Output {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let future = crate::util::trace::task(
+ future,
+ "block_on",
+ None,
+ crate::runtime::task::Id::next().as_u64(),
+ );
+
+ let _enter = self.enter();
+
+ match &self.scheduler {
+ Scheduler::CurrentThread(exec) => exec.block_on(&self.handle.inner, future),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Scheduler::MultiThread(exec) => exec.block_on(&self.handle.inner, future),
+ }
+ }
+
+ /// Enters the runtime context.
+ ///
+ /// This allows you to construct types that must have an executor
+ /// available on creation such as [`Sleep`] or [`TcpStream`]. It will
+ /// also allow you to call methods such as [`tokio::spawn`].
+ ///
+ /// [`Sleep`]: struct@crate::time::Sleep
+ /// [`TcpStream`]: struct@crate::net::TcpStream
+ /// [`tokio::spawn`]: fn@crate::spawn
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ ///
+ /// fn function_that_spawns(msg: String) {
+ /// // Had we not used `rt.enter` below, this would panic.
+ /// tokio::spawn(async move {
+ /// println!("{}", msg);
+ /// });
+ /// }
+ ///
+ /// fn main() {
+ /// let rt = Runtime::new().unwrap();
+ ///
+ /// let s = "Hello World!".to_string();
+ ///
+ /// // By entering the context, we tie `tokio::spawn` to this executor.
+ /// let _guard = rt.enter();
+ /// function_that_spawns(s);
+ /// }
+ /// ```
+ pub fn enter(&self) -> EnterGuard<'_> {
+ self.handle.enter()
+ }
+
+ /// Shuts down the runtime, waiting for at most `duration` for all spawned
+ /// task to shutdown.
+ ///
+ /// Usually, dropping a `Runtime` handle is sufficient as tasks are able to
+ /// shutdown in a timely fashion. However, dropping a `Runtime` will wait
+ /// indefinitely for all tasks to terminate, and there are cases where a long
+ /// blocking task has been spawned, which can block dropping `Runtime`.
+ ///
+ /// In this case, calling `shutdown_timeout` with an explicit wait timeout
+ /// can work. The `shutdown_timeout` will signal all tasks to shutdown and
+ /// will wait for at most `duration` for all spawned tasks to terminate. If
+ /// `timeout` elapses before all tasks are dropped, the function returns and
+ /// outstanding tasks are potentially leaked.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ /// use tokio::task;
+ ///
+ /// use std::thread;
+ /// use std::time::Duration;
+ ///
+ /// fn main() {
+ /// let runtime = Runtime::new().unwrap();
+ ///
+ /// runtime.block_on(async move {
+ /// task::spawn_blocking(move || {
+ /// thread::sleep(Duration::from_secs(10_000));
+ /// });
+ /// });
+ ///
+ /// runtime.shutdown_timeout(Duration::from_millis(100));
+ /// }
+ /// ```
+ pub fn shutdown_timeout(mut self, duration: Duration) {
+ // Wakeup and shutdown all the worker threads
+ self.handle.inner.shutdown();
+ self.blocking_pool.shutdown(Some(duration));
+ }
+
+ /// Shuts down the runtime, without waiting for any spawned tasks to shutdown.
+ ///
+ /// This can be useful if you want to drop a runtime from within another runtime.
+ /// Normally, dropping a runtime will block indefinitely for spawned blocking tasks
+ /// to complete, which would normally not be permitted within an asynchronous context.
+ /// By calling `shutdown_background()`, you can drop the runtime from such a context.
+ ///
+ /// Note however, that because we do not wait for any blocking tasks to complete, this
+ /// may result in a resource leak (in that any blocking tasks are still running until they
+ /// return.
+ ///
+ /// This function is equivalent to calling `shutdown_timeout(Duration::from_nanos(0))`.
+ ///
+ /// ```
+ /// use tokio::runtime::Runtime;
+ ///
+ /// fn main() {
+ /// let runtime = Runtime::new().unwrap();
+ ///
+ /// runtime.block_on(async move {
+ /// let inner_runtime = Runtime::new().unwrap();
+ /// // ...
+ /// inner_runtime.shutdown_background();
+ /// });
+ /// }
+ /// ```
+ pub fn shutdown_background(self) {
+ self.shutdown_timeout(Duration::from_nanos(0))
+ }
+}
+
+#[allow(clippy::single_match)] // there are comments in the error branch, so we don't want if-let
+impl Drop for Runtime {
+ fn drop(&mut self) {
+ match &mut self.scheduler {
+ Scheduler::CurrentThread(current_thread) => {
+ // This ensures that tasks spawned on the current-thread
+ // runtime are dropped inside the runtime's context.
+ let _guard = context::try_set_current(&self.handle.inner);
+ current_thread.shutdown(&self.handle.inner);
+ }
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Scheduler::MultiThread(multi_thread) => {
+ // The threaded scheduler drops its tasks on its worker threads, which is
+ // already in the runtime's context.
+ multi_thread.shutdown(&self.handle.inner);
+ }
+ }
+ }
+}
+
+cfg_metrics! {
+ impl Runtime {
+ /// TODO
+ pub fn metrics(&self) -> crate::runtime::RuntimeMetrics {
+ self.handle.metrics()
+ }
+ }
+}
diff --git a/src/runtime/scheduler/current_thread.rs b/src/runtime/scheduler/current_thread.rs
new file mode 100644
index 0000000..375e47c
--- /dev/null
+++ b/src/runtime/scheduler/current_thread.rs
@@ -0,0 +1,634 @@
+use crate::future::poll_fn;
+use crate::loom::sync::atomic::AtomicBool;
+use crate::loom::sync::{Arc, Mutex};
+use crate::runtime::driver::{self, Driver};
+use crate::runtime::task::{self, JoinHandle, OwnedTasks, Schedule, Task};
+use crate::runtime::{blocking, context, scheduler, Config};
+use crate::runtime::{MetricsBatch, SchedulerMetrics, WorkerMetrics};
+use crate::sync::notify::Notify;
+use crate::util::atomic_cell::AtomicCell;
+use crate::util::{waker_ref, RngSeedGenerator, Wake, WakerRef};
+
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use std::fmt;
+use std::future::Future;
+use std::sync::atomic::Ordering::{AcqRel, Release};
+use std::task::Poll::{Pending, Ready};
+use std::time::Duration;
+
+/// Executes tasks on the current thread
+pub(crate) struct CurrentThread {
+ /// Core scheduler data is acquired by a thread entering `block_on`.
+ core: AtomicCell<Core>,
+
+ /// Notifier for waking up other threads to steal the
+ /// driver.
+ notify: Notify,
+}
+
+/// Handle to the current thread scheduler
+pub(crate) struct Handle {
+ /// Scheduler state shared across threads
+ shared: Shared,
+
+ /// Resource driver handles
+ pub(crate) driver: driver::Handle,
+
+ /// Blocking pool spawner
+ pub(crate) blocking_spawner: blocking::Spawner,
+
+ /// Current random number generator seed
+ pub(crate) seed_generator: RngSeedGenerator,
+}
+
+/// Data required for executing the scheduler. The struct is passed around to
+/// a function that will perform the scheduling work and acts as a capability token.
+struct Core {
+ /// Scheduler run queue
+ tasks: VecDeque<task::Notified<Arc<Handle>>>,
+
+ /// Current tick
+ tick: u32,
+
+ /// Runtime driver
+ ///
+ /// The driver is removed before starting to park the thread
+ driver: Option<Driver>,
+
+ /// Metrics batch
+ metrics: MetricsBatch,
+
+ /// True if a task panicked without being handled and the runtime is
+ /// configured to shutdown on unhandled panic.
+ unhandled_panic: bool,
+}
+
+/// Scheduler state shared between threads.
+struct Shared {
+ /// Remote run queue. None if the `Runtime` has been dropped.
+ queue: Mutex<Option<VecDeque<task::Notified<Arc<Handle>>>>>,
+
+ /// Collection of all active tasks spawned onto this executor.
+ owned: OwnedTasks<Arc<Handle>>,
+
+ /// Indicates whether the blocked on thread was woken.
+ woken: AtomicBool,
+
+ /// Scheduler configuration options
+ config: Config,
+
+ /// Keeps track of various runtime metrics.
+ scheduler_metrics: SchedulerMetrics,
+
+ /// This scheduler only has one worker.
+ worker_metrics: WorkerMetrics,
+}
+
+/// Thread-local context.
+struct Context {
+ /// Scheduler handle
+ handle: Arc<Handle>,
+
+ /// Scheduler core, enabling the holder of `Context` to execute the
+ /// scheduler.
+ core: RefCell<Option<Box<Core>>>,
+}
+
+/// Initial queue capacity.
+const INITIAL_CAPACITY: usize = 64;
+
+// Tracks the current CurrentThread.
+scoped_thread_local!(static CURRENT: Context);
+
+impl CurrentThread {
+ pub(crate) fn new(
+ driver: Driver,
+ driver_handle: driver::Handle,
+ blocking_spawner: blocking::Spawner,
+ seed_generator: RngSeedGenerator,
+ config: Config,
+ ) -> (CurrentThread, Arc<Handle>) {
+ let handle = Arc::new(Handle {
+ shared: Shared {
+ queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))),
+ owned: OwnedTasks::new(),
+ woken: AtomicBool::new(false),
+ config,
+ scheduler_metrics: SchedulerMetrics::new(),
+ worker_metrics: WorkerMetrics::new(),
+ },
+ driver: driver_handle,
+ blocking_spawner,
+ seed_generator,
+ });
+
+ let core = AtomicCell::new(Some(Box::new(Core {
+ tasks: VecDeque::with_capacity(INITIAL_CAPACITY),
+ tick: 0,
+ driver: Some(driver),
+ metrics: MetricsBatch::new(),
+ unhandled_panic: false,
+ })));
+
+ let scheduler = CurrentThread {
+ core,
+ notify: Notify::new(),
+ };
+
+ (scheduler, handle)
+ }
+
+ #[track_caller]
+ pub(crate) fn block_on<F: Future>(&self, handle: &scheduler::Handle, future: F) -> F::Output {
+ pin!(future);
+
+ let mut enter = crate::runtime::context::enter_runtime(handle, false);
+ let handle = handle.as_current_thread();
+
+ // Attempt to steal the scheduler core and block_on the future if we can
+ // there, otherwise, lets select on a notification that the core is
+ // available or the future is complete.
+ loop {
+ if let Some(core) = self.take_core(handle) {
+ return core.block_on(future);
+ } else {
+ let notified = self.notify.notified();
+ pin!(notified);
+
+ if let Some(out) = enter
+ .blocking
+ .block_on(poll_fn(|cx| {
+ if notified.as_mut().poll(cx).is_ready() {
+ return Ready(None);
+ }
+
+ if let Ready(out) = future.as_mut().poll(cx) {
+ return Ready(Some(out));
+ }
+
+ Pending
+ }))
+ .expect("Failed to `Enter::block_on`")
+ {
+ return out;
+ }
+ }
+ }
+ }
+
+ fn take_core(&self, handle: &Arc<Handle>) -> Option<CoreGuard<'_>> {
+ let core = self.core.take()?;
+
+ Some(CoreGuard {
+ context: Context {
+ handle: handle.clone(),
+ core: RefCell::new(Some(core)),
+ },
+ scheduler: self,
+ })
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) {
+ let handle = handle.as_current_thread();
+
+ // Avoid a double panic if we are currently panicking and
+ // the lock may be poisoned.
+
+ let core = match self.take_core(handle) {
+ Some(core) => core,
+ None if std::thread::panicking() => return,
+ None => panic!("Oh no! We never placed the Core back, this is a bug!"),
+ };
+
+ core.enter(|mut core, _context| {
+ // Drain the OwnedTasks collection. This call also closes the
+ // collection, ensuring that no tasks are ever pushed after this
+ // call returns.
+ handle.shared.owned.close_and_shutdown_all();
+
+ // Drain local queue
+ // We already shut down every task, so we just need to drop the task.
+ while let Some(task) = core.pop_task(handle) {
+ drop(task);
+ }
+
+ // Drain remote queue and set it to None
+ let remote_queue = handle.shared.queue.lock().take();
+
+ // Using `Option::take` to replace the shared queue with `None`.
+ // We already shut down every task, so we just need to drop the task.
+ if let Some(remote_queue) = remote_queue {
+ for task in remote_queue {
+ drop(task);
+ }
+ }
+
+ assert!(handle.shared.owned.is_empty());
+
+ // Submit metrics
+ core.metrics.submit(&handle.shared.worker_metrics);
+
+ // Shutdown the resource drivers
+ if let Some(driver) = core.driver.as_mut() {
+ driver.shutdown(&handle.driver);
+ }
+
+ (core, ())
+ });
+ }
+}
+
+impl fmt::Debug for CurrentThread {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt.debug_struct("CurrentThread").finish()
+ }
+}
+
+// ===== impl Core =====
+
+impl Core {
+ fn pop_task(&mut self, handle: &Handle) -> Option<task::Notified<Arc<Handle>>> {
+ let ret = self.tasks.pop_front();
+ handle
+ .shared
+ .worker_metrics
+ .set_queue_depth(self.tasks.len());
+ ret
+ }
+
+ fn push_task(&mut self, handle: &Handle, task: task::Notified<Arc<Handle>>) {
+ self.tasks.push_back(task);
+ self.metrics.inc_local_schedule_count();
+ handle
+ .shared
+ .worker_metrics
+ .set_queue_depth(self.tasks.len());
+ }
+}
+
+fn did_defer_tasks() -> bool {
+ context::with_defer(|deferred| !deferred.is_empty()).unwrap()
+}
+
+fn wake_deferred_tasks() {
+ context::with_defer(|deferred| deferred.wake());
+}
+
+// ===== impl Context =====
+
+impl Context {
+ /// Execute the closure with the given scheduler core stored in the
+ /// thread-local context.
+ fn run_task<R>(&self, mut core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) {
+ core.metrics.incr_poll_count();
+ self.enter(core, || crate::runtime::coop::budget(f))
+ }
+
+ /// Blocks the current thread until an event is received by the driver,
+ /// including I/O events, timer events, ...
+ fn park(&self, mut core: Box<Core>, handle: &Handle) -> Box<Core> {
+ let mut driver = core.driver.take().expect("driver missing");
+
+ if let Some(f) = &handle.shared.config.before_park {
+ // Incorrect lint, the closures are actually different types so `f`
+ // cannot be passed as an argument to `enter`.
+ #[allow(clippy::redundant_closure)]
+ let (c, _) = self.enter(core, || f());
+ core = c;
+ }
+
+ // This check will fail if `before_park` spawns a task for us to run
+ // instead of parking the thread
+ if core.tasks.is_empty() {
+ // Park until the thread is signaled
+ core.metrics.about_to_park();
+ core.metrics.submit(&handle.shared.worker_metrics);
+
+ let (c, _) = self.enter(core, || {
+ driver.park(&handle.driver);
+ wake_deferred_tasks();
+ });
+
+ core = c;
+ core.metrics.returned_from_park();
+ }
+
+ if let Some(f) = &handle.shared.config.after_unpark {
+ // Incorrect lint, the closures are actually different types so `f`
+ // cannot be passed as an argument to `enter`.
+ #[allow(clippy::redundant_closure)]
+ let (c, _) = self.enter(core, || f());
+ core = c;
+ }
+
+ core.driver = Some(driver);
+ core
+ }
+
+ /// Checks the driver for new events without blocking the thread.
+ fn park_yield(&self, mut core: Box<Core>, handle: &Handle) -> Box<Core> {
+ let mut driver = core.driver.take().expect("driver missing");
+
+ core.metrics.submit(&handle.shared.worker_metrics);
+ let (mut core, _) = self.enter(core, || {
+ driver.park_timeout(&handle.driver, Duration::from_millis(0));
+ wake_deferred_tasks();
+ });
+
+ core.driver = Some(driver);
+ core
+ }
+
+ fn enter<R>(&self, core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) {
+ // Store the scheduler core in the thread-local context
+ //
+ // A drop-guard is employed at a higher level.
+ *self.core.borrow_mut() = Some(core);
+
+ // Execute the closure while tracking the execution budget
+ let ret = f();
+
+ // Take the scheduler core back
+ let core = self.core.borrow_mut().take().expect("core missing");
+ (core, ret)
+ }
+}
+
+// ===== impl Handle =====
+
+impl Handle {
+ /// Spawns a future onto the `CurrentThread` scheduler
+ pub(crate) fn spawn<F>(
+ me: &Arc<Self>,
+ future: F,
+ id: crate::runtime::task::Id,
+ ) -> JoinHandle<F::Output>
+ where
+ F: crate::future::Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ let (handle, notified) = me.shared.owned.bind(future, me.clone(), id);
+
+ if let Some(notified) = notified {
+ me.schedule(notified);
+ }
+
+ handle
+ }
+
+ fn pop(&self) -> Option<task::Notified<Arc<Handle>>> {
+ match self.shared.queue.lock().as_mut() {
+ Some(queue) => queue.pop_front(),
+ None => None,
+ }
+ }
+
+ fn waker_ref(me: &Arc<Self>) -> WakerRef<'_> {
+ // Set woken to true when enter block_on, ensure outer future
+ // be polled for the first time when enter loop
+ me.shared.woken.store(true, Release);
+ waker_ref(me)
+ }
+
+ // reset woken to false and return original value
+ pub(crate) fn reset_woken(&self) -> bool {
+ self.shared.woken.swap(false, AcqRel)
+ }
+}
+
+cfg_metrics! {
+ impl Handle {
+ pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
+ &self.shared.scheduler_metrics
+ }
+
+ pub(crate) fn injection_queue_depth(&self) -> usize {
+ // TODO: avoid having to lock. The multi-threaded injection queue
+ // could probably be used here.
+ self.shared
+ .queue
+ .lock()
+ .as_ref()
+ .map(|queue| queue.len())
+ .unwrap_or(0)
+ }
+
+ pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
+ assert_eq!(0, worker);
+ &self.shared.worker_metrics
+ }
+
+ pub(crate) fn num_blocking_threads(&self) -> usize {
+ self.blocking_spawner.num_threads()
+ }
+
+ pub(crate) fn num_idle_blocking_threads(&self) -> usize {
+ self.blocking_spawner.num_idle_threads()
+ }
+
+ pub(crate) fn blocking_queue_depth(&self) -> usize {
+ self.blocking_spawner.queue_depth()
+ }
+ }
+}
+
+impl fmt::Debug for Handle {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt.debug_struct("current_thread::Handle { ... }").finish()
+ }
+}
+
+// ===== impl Shared =====
+
+impl Schedule for Arc<Handle> {
+ fn release(&self, task: &Task<Self>) -> Option<Task<Self>> {
+ self.shared.owned.remove(task)
+ }
+
+ fn schedule(&self, task: task::Notified<Self>) {
+ CURRENT.with(|maybe_cx| match maybe_cx {
+ Some(cx) if Arc::ptr_eq(self, &cx.handle) => {
+ let mut core = cx.core.borrow_mut();
+
+ // If `None`, the runtime is shutting down, so there is no need
+ // to schedule the task.
+ if let Some(core) = core.as_mut() {
+ core.push_task(self, task);
+ }
+ }
+ _ => {
+ // Track that a task was scheduled from **outside** of the runtime.
+ self.shared.scheduler_metrics.inc_remote_schedule_count();
+
+ // If the queue is None, then the runtime has shut down. We
+ // don't need to do anything with the notification in that case.
+ let mut guard = self.shared.queue.lock();
+ if let Some(queue) = guard.as_mut() {
+ queue.push_back(task);
+ drop(guard);
+ self.driver.unpark();
+ }
+ }
+ });
+ }
+
+ cfg_unstable! {
+ fn unhandled_panic(&self) {
+ use crate::runtime::UnhandledPanic;
+
+ match self.shared.config.unhandled_panic {
+ UnhandledPanic::Ignore => {
+ // Do nothing
+ }
+ UnhandledPanic::ShutdownRuntime => {
+ // This hook is only called from within the runtime, so
+ // `CURRENT` should match with `&self`, i.e. there is no
+ // opportunity for a nested scheduler to be called.
+ CURRENT.with(|maybe_cx| match maybe_cx {
+ Some(cx) if Arc::ptr_eq(self, &cx.handle) => {
+ let mut core = cx.core.borrow_mut();
+
+ // If `None`, the runtime is shutting down, so there is no need to signal shutdown
+ if let Some(core) = core.as_mut() {
+ core.unhandled_panic = true;
+ self.shared.owned.close_and_shutdown_all();
+ }
+ }
+ _ => unreachable!("runtime core not set in CURRENT thread-local"),
+ })
+ }
+ }
+ }
+ }
+}
+
+impl Wake for Handle {
+ fn wake(arc_self: Arc<Self>) {
+ Wake::wake_by_ref(&arc_self)
+ }
+
+ /// Wake by reference
+ fn wake_by_ref(arc_self: &Arc<Self>) {
+ arc_self.shared.woken.store(true, Release);
+ arc_self.driver.unpark();
+ }
+}
+
+// ===== CoreGuard =====
+
+/// Used to ensure we always place the `Core` value back into its slot in
+/// `CurrentThread`, even if the future panics.
+struct CoreGuard<'a> {
+ context: Context,
+ scheduler: &'a CurrentThread,
+}
+
+impl CoreGuard<'_> {
+ #[track_caller]
+ fn block_on<F: Future>(self, future: F) -> F::Output {
+ let ret = self.enter(|mut core, context| {
+ let waker = Handle::waker_ref(&context.handle);
+ let mut cx = std::task::Context::from_waker(&waker);
+
+ pin!(future);
+
+ 'outer: loop {
+ let handle = &context.handle;
+
+ if handle.reset_woken() {
+ let (c, res) = context.enter(core, || {
+ crate::runtime::coop::budget(|| future.as_mut().poll(&mut cx))
+ });
+
+ core = c;
+
+ if let Ready(v) = res {
+ return (core, Some(v));
+ }
+ }
+
+ for _ in 0..handle.shared.config.event_interval {
+ // Make sure we didn't hit an unhandled_panic
+ if core.unhandled_panic {
+ return (core, None);
+ }
+
+ // Get and increment the current tick
+ let tick = core.tick;
+ core.tick = core.tick.wrapping_add(1);
+
+ let entry = if tick % handle.shared.config.global_queue_interval == 0 {
+ handle.pop().or_else(|| core.tasks.pop_front())
+ } else {
+ core.tasks.pop_front().or_else(|| handle.pop())
+ };
+
+ let task = match entry {
+ Some(entry) => entry,
+ None => {
+ core = if did_defer_tasks() {
+ context.park_yield(core, handle)
+ } else {
+ context.park(core, handle)
+ };
+
+ // Try polling the `block_on` future next
+ continue 'outer;
+ }
+ };
+
+ let task = context.handle.shared.owned.assert_owner(task);
+
+ let (c, _) = context.run_task(core, || {
+ task.run();
+ });
+
+ core = c;
+ }
+
+ // Yield to the driver, this drives the timer and pulls any
+ // pending I/O events.
+ core = context.park_yield(core, handle);
+ }
+ });
+
+ match ret {
+ Some(ret) => ret,
+ None => {
+ // `block_on` panicked.
+ panic!("a spawned task panicked and the runtime is configured to shut down on unhandled panic");
+ }
+ }
+ }
+
+ /// Enters the scheduler context. This sets the queue and other necessary
+ /// scheduler state in the thread-local.
+ fn enter<F, R>(self, f: F) -> R
+ where
+ F: FnOnce(Box<Core>, &Context) -> (Box<Core>, R),
+ {
+ // Remove `core` from `context` to pass into the closure.
+ let core = self.context.core.borrow_mut().take().expect("core missing");
+
+ // Call the closure and place `core` back
+ let (core, ret) = CURRENT.set(&self.context, || f(core, &self.context));
+
+ *self.context.core.borrow_mut() = Some(core);
+
+ ret
+ }
+}
+
+impl Drop for CoreGuard<'_> {
+ fn drop(&mut self) {
+ if let Some(core) = self.context.core.borrow_mut().take() {
+ // Replace old scheduler back into the state to allow
+ // other threads to pick it up and drive it.
+ self.scheduler.core.set(core);
+
+ // Wake up other possible threads that could steal the driver.
+ self.scheduler.notify.notify_one()
+ }
+ }
+}
diff --git a/src/runtime/scheduler/mod.rs b/src/runtime/scheduler/mod.rs
new file mode 100644
index 0000000..f45d8a8
--- /dev/null
+++ b/src/runtime/scheduler/mod.rs
@@ -0,0 +1,194 @@
+cfg_rt! {
+ pub(crate) mod current_thread;
+ pub(crate) use current_thread::CurrentThread;
+}
+
+cfg_rt_multi_thread! {
+ pub(crate) mod multi_thread;
+ pub(crate) use multi_thread::MultiThread;
+}
+
+use crate::runtime::driver;
+
+#[derive(Debug, Clone)]
+pub(crate) enum Handle {
+ #[cfg(feature = "rt")]
+ CurrentThread(Arc<current_thread::Handle>),
+
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ MultiThread(Arc<multi_thread::Handle>),
+
+ // TODO: This is to avoid triggering "dead code" warnings many other places
+ // in the codebase. Remove this during a later cleanup
+ #[cfg(not(feature = "rt"))]
+ #[allow(dead_code)]
+ Disabled,
+}
+
+impl Handle {
+ #[cfg_attr(not(feature = "full"), allow(dead_code))]
+ pub(crate) fn driver(&self) -> &driver::Handle {
+ match *self {
+ #[cfg(feature = "rt")]
+ Handle::CurrentThread(ref h) => &h.driver,
+
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(ref h) => &h.driver,
+
+ #[cfg(not(feature = "rt"))]
+ Handle::Disabled => unreachable!(),
+ }
+ }
+}
+
+cfg_rt! {
+ use crate::future::Future;
+ use crate::loom::sync::Arc;
+ use crate::runtime::{blocking, task::Id};
+ use crate::runtime::context;
+ use crate::task::JoinHandle;
+ use crate::util::RngSeedGenerator;
+
+ impl Handle {
+ #[track_caller]
+ pub(crate) fn current() -> Handle {
+ match context::try_current() {
+ Ok(handle) => handle,
+ Err(e) => panic!("{}", e),
+ }
+ }
+
+ pub(crate) fn blocking_spawner(&self) -> &blocking::Spawner {
+ match self {
+ Handle::CurrentThread(h) => &h.blocking_spawner,
+
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(h) => &h.blocking_spawner,
+ }
+ }
+
+ pub(crate) fn spawn<F>(&self, future: F, id: Id) -> JoinHandle<F::Output>
+ where
+ F: Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ match self {
+ Handle::CurrentThread(h) => current_thread::Handle::spawn(h, future, id),
+
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(h) => multi_thread::Handle::spawn(h, future, id),
+ }
+ }
+
+ pub(crate) fn shutdown(&self) {
+ match *self {
+ Handle::CurrentThread(_) => {},
+
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(ref h) => h.shutdown(),
+ }
+ }
+
+ pub(crate) fn seed_generator(&self) -> &RngSeedGenerator {
+ match self {
+ Handle::CurrentThread(h) => &h.seed_generator,
+
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(h) => &h.seed_generator,
+ }
+ }
+
+ pub(crate) fn as_current_thread(&self) -> &Arc<current_thread::Handle> {
+ match self {
+ Handle::CurrentThread(handle) => handle,
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ _ => panic!("not a CurrentThread handle"),
+ }
+ }
+ }
+
+ cfg_metrics! {
+ use crate::runtime::{SchedulerMetrics, WorkerMetrics};
+
+ impl Handle {
+ pub(crate) fn num_workers(&self) -> usize {
+ match self {
+ Handle::CurrentThread(_) => 1,
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.num_workers(),
+ }
+ }
+
+ pub(crate) fn num_blocking_threads(&self) -> usize {
+ match self {
+ Handle::CurrentThread(handle) => handle.num_blocking_threads(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.num_blocking_threads(),
+ }
+ }
+
+ pub(crate) fn num_idle_blocking_threads(&self) -> usize {
+ match self {
+ Handle::CurrentThread(handle) => handle.num_idle_blocking_threads(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.num_idle_blocking_threads(),
+ }
+ }
+
+ pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
+ match self {
+ Handle::CurrentThread(handle) => handle.scheduler_metrics(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.scheduler_metrics(),
+ }
+ }
+
+ pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
+ match self {
+ Handle::CurrentThread(handle) => handle.worker_metrics(worker),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.worker_metrics(worker),
+ }
+ }
+
+ pub(crate) fn injection_queue_depth(&self) -> usize {
+ match self {
+ Handle::CurrentThread(handle) => handle.injection_queue_depth(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.injection_queue_depth(),
+ }
+ }
+
+ pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+ match self {
+ Handle::CurrentThread(handle) => handle.worker_metrics(worker).queue_depth(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.worker_local_queue_depth(worker),
+ }
+ }
+
+ pub(crate) fn blocking_queue_depth(&self) -> usize {
+ match self {
+ Handle::CurrentThread(handle) => handle.blocking_queue_depth(),
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ Handle::MultiThread(handle) => handle.blocking_queue_depth(),
+ }
+ }
+ }
+ }
+}
+
+cfg_not_rt! {
+ #[cfg(any(
+ feature = "net",
+ all(unix, feature = "process"),
+ all(unix, feature = "signal"),
+ feature = "time",
+ ))]
+ impl Handle {
+ #[track_caller]
+ pub(crate) fn current() -> Handle {
+ panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR)
+ }
+ }
+}
diff --git a/src/runtime/scheduler/multi_thread/handle.rs b/src/runtime/scheduler/multi_thread/handle.rs
new file mode 100644
index 0000000..69a4620
--- /dev/null
+++ b/src/runtime/scheduler/multi_thread/handle.rs
@@ -0,0 +1,98 @@
+use crate::future::Future;
+use crate::loom::sync::Arc;
+use crate::runtime::scheduler::multi_thread::worker;
+use crate::runtime::{
+ blocking, driver,
+ task::{self, JoinHandle},
+};
+use crate::util::RngSeedGenerator;
+
+use std::fmt;
+
+/// Handle to the multi thread scheduler
+pub(crate) struct Handle {
+ /// Task spawner
+ pub(super) shared: worker::Shared,
+
+ /// Resource driver handles
+ pub(crate) driver: driver::Handle,
+
+ /// Blocking pool spawner
+ pub(crate) blocking_spawner: blocking::Spawner,
+
+ /// Current random number generator seed
+ pub(crate) seed_generator: RngSeedGenerator,
+}
+
+impl Handle {
+ /// Spawns a future onto the thread pool
+ pub(crate) fn spawn<F>(me: &Arc<Self>, future: F, id: task::Id) -> JoinHandle<F::Output>
+ where
+ F: crate::future::Future + Send + 'static,
+ F::Output: Send + 'static,
+ {
+ Self::bind_new_task(me, future, id)
+ }
+
+ pub(crate) fn shutdown(&self) {
+ self.close();
+ }
+
+ pub(super) fn bind_new_task<T>(me: &Arc<Self>, future: T, id: task::Id) -> JoinHandle<T::Output>
+ where
+ T: Future + Send + 'static,
+ T::Output: Send + 'static,
+ {
+ let (handle, notified) = me.shared.owned.bind(future, me.clone(), id);
+
+ if let Some(notified) = notified {
+ me.schedule_task(notified, false);
+ }
+
+ handle
+ }
+}
+
+cfg_metrics! {
+ use crate::runtime::{SchedulerMetrics, WorkerMetrics};
+
+ impl Handle {
+ pub(crate) fn num_workers(&self) -> usize {
+ self.shared.worker_metrics.len()
+ }
+
+ pub(crate) fn num_blocking_threads(&self) -> usize {
+ self.blocking_spawner.num_threads()
+ }
+
+ pub(crate) fn num_idle_blocking_threads(&self) -> usize {
+ self.blocking_spawner.num_idle_threads()
+ }
+
+ pub(crate) fn scheduler_metrics(&self) -> &SchedulerMetrics {
+ &self.shared.scheduler_metrics
+ }
+
+ pub(crate) fn worker_metrics(&self, worker: usize) -> &WorkerMetrics {
+ &self.shared.worker_metrics[worker]
+ }
+
+ pub(crate) fn injection_queue_depth(&self) -> usize {
+ self.shared.injection_queue_depth()
+ }
+
+ pub(crate) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+ self.shared.worker_local_queue_depth(worker)
+ }
+
+ pub(crate) fn blocking_queue_depth(&self) -> usize {
+ self.blocking_spawner.queue_depth()
+ }
+ }
+}
+
+impl fmt::Debug for Handle {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt.debug_struct("multi_thread::Handle { ... }").finish()
+ }
+}
diff --git a/src/runtime/thread_pool/idle.rs b/src/runtime/scheduler/multi_thread/idle.rs
similarity index 93%
rename from src/runtime/thread_pool/idle.rs
rename to src/runtime/scheduler/multi_thread/idle.rs
index 6b7ee12..a57bf6a 100644
--- a/src/runtime/thread_pool/idle.rs
+++ b/src/runtime/scheduler/multi_thread/idle.rs
@@ -64,7 +64,7 @@
// A worker should be woken up, atomically increment the number of
// searching workers as well as the number of unparked workers.
- State::unpark_one(&self.state);
+ State::unpark_one(&self.state, 1);
// Get the worker to unpark
let ret = sleepers.pop();
@@ -111,7 +111,9 @@
/// Unpark a specific worker. This happens if tasks are submitted from
/// within the worker's park routine.
- pub(super) fn unpark_worker_by_id(&self, worker_id: usize) {
+ ///
+ /// Returns `true` if the worker was parked before calling the method.
+ pub(super) fn unpark_worker_by_id(&self, worker_id: usize) -> bool {
let mut sleepers = self.sleepers.lock();
for index in 0..sleepers.len() {
@@ -119,11 +121,13 @@
sleepers.swap_remove(index);
// Update the state accordingly while the lock is held.
- State::unpark_one(&self.state);
+ State::unpark_one(&self.state, 0);
- return;
+ return true;
}
}
+
+ false
}
/// Returns `true` if `worker_id` is contained in the sleep set.
@@ -151,8 +155,8 @@
State(cell.load(ordering))
}
- fn unpark_one(cell: &AtomicUsize) {
- cell.fetch_add(1 | (1 << UNPARK_SHIFT), SeqCst);
+ fn unpark_one(cell: &AtomicUsize, num_searching: usize) {
+ cell.fetch_add(num_searching | (1 << UNPARK_SHIFT), SeqCst);
}
fn inc_num_searching(cell: &AtomicUsize, ordering: Ordering) {
diff --git a/src/runtime/scheduler/multi_thread/mod.rs b/src/runtime/scheduler/multi_thread/mod.rs
new file mode 100644
index 0000000..47cd1f3
--- /dev/null
+++ b/src/runtime/scheduler/multi_thread/mod.rs
@@ -0,0 +1,84 @@
+//! Multi-threaded runtime
+
+mod handle;
+pub(crate) use handle::Handle;
+
+mod idle;
+use self::idle::Idle;
+
+mod park;
+pub(crate) use park::{Parker, Unparker};
+
+pub(crate) mod queue;
+
+mod worker;
+pub(crate) use worker::Launch;
+
+pub(crate) use worker::block_in_place;
+
+use crate::loom::sync::Arc;
+use crate::runtime::{
+ blocking,
+ driver::{self, Driver},
+ scheduler, Config,
+};
+use crate::util::RngSeedGenerator;
+
+use std::fmt;
+use std::future::Future;
+
+/// Work-stealing based thread pool for executing futures.
+pub(crate) struct MultiThread;
+
+// ===== impl MultiThread =====
+
+impl MultiThread {
+ pub(crate) fn new(
+ size: usize,
+ driver: Driver,
+ driver_handle: driver::Handle,
+ blocking_spawner: blocking::Spawner,
+ seed_generator: RngSeedGenerator,
+ config: Config,
+ ) -> (MultiThread, Arc<Handle>, Launch) {
+ let parker = Parker::new(driver);
+ let (handle, launch) = worker::create(
+ size,
+ parker,
+ driver_handle,
+ blocking_spawner,
+ seed_generator,
+ config,
+ );
+
+ (MultiThread, handle, launch)
+ }
+
+ /// Blocks the current thread waiting for the future to complete.
+ ///
+ /// The future will execute on the current thread, but all spawned tasks
+ /// will be executed on the thread pool.
+ pub(crate) fn block_on<F>(&self, handle: &scheduler::Handle, future: F) -> F::Output
+ where
+ F: Future,
+ {
+ let mut enter = crate::runtime::context::enter_runtime(handle, true);
+ enter
+ .blocking
+ .block_on(future)
+ .expect("failed to park thread")
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &scheduler::Handle) {
+ match handle {
+ scheduler::Handle::MultiThread(handle) => handle.shutdown(),
+ _ => panic!("expected MultiThread scheduler"),
+ }
+ }
+}
+
+impl fmt::Debug for MultiThread {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt.debug_struct("MultiThread").finish()
+ }
+}
diff --git a/src/runtime/scheduler/multi_thread/park.rs b/src/runtime/scheduler/multi_thread/park.rs
new file mode 100644
index 0000000..6bdbff9
--- /dev/null
+++ b/src/runtime/scheduler/multi_thread/park.rs
@@ -0,0 +1,237 @@
+//! Parks the runtime.
+//!
+//! A combination of the various resource driver park handles.
+
+use crate::loom::sync::atomic::AtomicUsize;
+use crate::loom::sync::{Arc, Condvar, Mutex};
+use crate::loom::thread;
+use crate::runtime::driver::{self, Driver};
+use crate::util::TryLock;
+
+use std::sync::atomic::Ordering::SeqCst;
+use std::time::Duration;
+
+pub(crate) struct Parker {
+ inner: Arc<Inner>,
+}
+
+pub(crate) struct Unparker {
+ inner: Arc<Inner>,
+}
+
+struct Inner {
+ /// Avoids entering the park if possible
+ state: AtomicUsize,
+
+ /// Used to coordinate access to the driver / condvar
+ mutex: Mutex<()>,
+
+ /// Condvar to block on if the driver is unavailable.
+ condvar: Condvar,
+
+ /// Resource (I/O, time, ...) driver
+ shared: Arc<Shared>,
+}
+
+const EMPTY: usize = 0;
+const PARKED_CONDVAR: usize = 1;
+const PARKED_DRIVER: usize = 2;
+const NOTIFIED: usize = 3;
+
+/// Shared across multiple Parker handles
+struct Shared {
+ /// Shared driver. Only one thread at a time can use this
+ driver: TryLock<Driver>,
+}
+
+impl Parker {
+ pub(crate) fn new(driver: Driver) -> Parker {
+ Parker {
+ inner: Arc::new(Inner {
+ state: AtomicUsize::new(EMPTY),
+ mutex: Mutex::new(()),
+ condvar: Condvar::new(),
+ shared: Arc::new(Shared {
+ driver: TryLock::new(driver),
+ }),
+ }),
+ }
+ }
+
+ pub(crate) fn unpark(&self) -> Unparker {
+ Unparker {
+ inner: self.inner.clone(),
+ }
+ }
+
+ pub(crate) fn park(&mut self, handle: &driver::Handle) {
+ self.inner.park(handle);
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) {
+ // Only parking with zero is supported...
+ assert_eq!(duration, Duration::from_millis(0));
+
+ if let Some(mut driver) = self.inner.shared.driver.try_lock() {
+ driver.park_timeout(handle, duration)
+ }
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &driver::Handle) {
+ self.inner.shutdown(handle);
+ }
+}
+
+impl Clone for Parker {
+ fn clone(&self) -> Parker {
+ Parker {
+ inner: Arc::new(Inner {
+ state: AtomicUsize::new(EMPTY),
+ mutex: Mutex::new(()),
+ condvar: Condvar::new(),
+ shared: self.inner.shared.clone(),
+ }),
+ }
+ }
+}
+
+impl Unparker {
+ pub(crate) fn unpark(&self, driver: &driver::Handle) {
+ self.inner.unpark(driver);
+ }
+}
+
+impl Inner {
+ /// Parks the current thread for at most `dur`.
+ fn park(&self, handle: &driver::Handle) {
+ for _ in 0..3 {
+ // If we were previously notified then we consume this notification and
+ // return quickly.
+ if self
+ .state
+ .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
+ .is_ok()
+ {
+ return;
+ }
+
+ thread::yield_now();
+ }
+
+ if let Some(mut driver) = self.shared.driver.try_lock() {
+ self.park_driver(&mut driver, handle);
+ } else {
+ self.park_condvar();
+ }
+ }
+
+ fn park_condvar(&self) {
+ // Otherwise we need to coordinate going to sleep
+ let mut m = self.mutex.lock();
+
+ match self
+ .state
+ .compare_exchange(EMPTY, PARKED_CONDVAR, SeqCst, SeqCst)
+ {
+ Ok(_) => {}
+ Err(NOTIFIED) => {
+ // We must read here, even though we know it will be `NOTIFIED`.
+ // This is because `unpark` may have been called again since we read
+ // `NOTIFIED` in the `compare_exchange` above. We must perform an
+ // acquire operation that synchronizes with that `unpark` to observe
+ // any writes it made before the call to unpark. To do that we must
+ // read from the write it made to `state`.
+ let old = self.state.swap(EMPTY, SeqCst);
+ debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly");
+
+ return;
+ }
+ Err(actual) => panic!("inconsistent park state; actual = {}", actual),
+ }
+
+ loop {
+ m = self.condvar.wait(m).unwrap();
+
+ if self
+ .state
+ .compare_exchange(NOTIFIED, EMPTY, SeqCst, SeqCst)
+ .is_ok()
+ {
+ // got a notification
+ return;
+ }
+
+ // spurious wakeup, go back to sleep
+ }
+ }
+
+ fn park_driver(&self, driver: &mut Driver, handle: &driver::Handle) {
+ match self
+ .state
+ .compare_exchange(EMPTY, PARKED_DRIVER, SeqCst, SeqCst)
+ {
+ Ok(_) => {}
+ Err(NOTIFIED) => {
+ // We must read here, even though we know it will be `NOTIFIED`.
+ // This is because `unpark` may have been called again since we read
+ // `NOTIFIED` in the `compare_exchange` above. We must perform an
+ // acquire operation that synchronizes with that `unpark` to observe
+ // any writes it made before the call to unpark. To do that we must
+ // read from the write it made to `state`.
+ let old = self.state.swap(EMPTY, SeqCst);
+ debug_assert_eq!(old, NOTIFIED, "park state changed unexpectedly");
+
+ return;
+ }
+ Err(actual) => panic!("inconsistent park state; actual = {}", actual),
+ }
+
+ driver.park(handle);
+
+ match self.state.swap(EMPTY, SeqCst) {
+ NOTIFIED => {} // got a notification, hurray!
+ PARKED_DRIVER => {} // no notification, alas
+ n => panic!("inconsistent park_timeout state: {}", n),
+ }
+ }
+
+ fn unpark(&self, driver: &driver::Handle) {
+ // To ensure the unparked thread will observe any writes we made before
+ // this call, we must perform a release operation that `park` can
+ // synchronize with. To do that we must write `NOTIFIED` even if `state`
+ // is already `NOTIFIED`. That is why this must be a swap rather than a
+ // compare-and-swap that returns if it reads `NOTIFIED` on failure.
+ match self.state.swap(NOTIFIED, SeqCst) {
+ EMPTY => {} // no one was waiting
+ NOTIFIED => {} // already unparked
+ PARKED_CONDVAR => self.unpark_condvar(),
+ PARKED_DRIVER => driver.unpark(),
+ actual => panic!("inconsistent state in unpark; actual = {}", actual),
+ }
+ }
+
+ fn unpark_condvar(&self) {
+ // There is a period between when the parked thread sets `state` to
+ // `PARKED` (or last checked `state` in the case of a spurious wake
+ // up) and when it actually waits on `cvar`. If we were to notify
+ // during this period it would be ignored and then when the parked
+ // thread went to sleep it would never wake up. Fortunately, it has
+ // `lock` locked at this stage so we can acquire `lock` to wait until
+ // it is ready to receive the notification.
+ //
+ // Releasing `lock` before the call to `notify_one` means that when the
+ // parked thread wakes it doesn't get woken only to have to wait for us
+ // to release `lock`.
+ drop(self.mutex.lock());
+
+ self.condvar.notify_one()
+ }
+
+ fn shutdown(&self, handle: &driver::Handle) {
+ if let Some(mut driver) = self.shared.driver.try_lock() {
+ driver.shutdown(handle);
+ }
+
+ self.condvar.notify_all();
+ }
+}
diff --git a/src/runtime/queue.rs b/src/runtime/scheduler/multi_thread/queue.rs
similarity index 79%
rename from src/runtime/queue.rs
rename to src/runtime/scheduler/multi_thread/queue.rs
index a88dffc..faf56db 100644
--- a/src/runtime/queue.rs
+++ b/src/runtime/scheduler/multi_thread/queue.rs
@@ -1,39 +1,56 @@
//! Run-queue structures to support a work-stealing scheduler
use crate::loom::cell::UnsafeCell;
-use crate::loom::sync::atomic::{AtomicU16, AtomicU32};
use crate::loom::sync::Arc;
-use crate::runtime::stats::WorkerStatsBatcher;
use crate::runtime::task::{self, Inject};
+use crate::runtime::MetricsBatch;
-use std::mem::MaybeUninit;
+use std::mem::{self, MaybeUninit};
use std::ptr;
use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release};
+// Use wider integers when possible to increase ABA resilience.
+//
+// See issue #5041: <https://github.com/tokio-rs/tokio/issues/5041>.
+cfg_has_atomic_u64! {
+ type UnsignedShort = u32;
+ type UnsignedLong = u64;
+ type AtomicUnsignedShort = crate::loom::sync::atomic::AtomicU32;
+ type AtomicUnsignedLong = crate::loom::sync::atomic::AtomicU64;
+}
+cfg_not_has_atomic_u64! {
+ type UnsignedShort = u16;
+ type UnsignedLong = u32;
+ type AtomicUnsignedShort = crate::loom::sync::atomic::AtomicU16;
+ type AtomicUnsignedLong = crate::loom::sync::atomic::AtomicU32;
+}
+
/// Producer handle. May only be used from a single thread.
-pub(super) struct Local<T: 'static> {
+pub(crate) struct Local<T: 'static> {
inner: Arc<Inner<T>>,
}
/// Consumer handle. May be used from many threads.
-pub(super) struct Steal<T: 'static>(Arc<Inner<T>>);
+pub(crate) struct Steal<T: 'static>(Arc<Inner<T>>);
-pub(super) struct Inner<T: 'static> {
+pub(crate) struct Inner<T: 'static> {
/// Concurrently updated by many threads.
///
- /// Contains two `u16` values. The LSB byte is the "real" head of the queue.
- /// The `u16` in the MSB is set by a stealer in process of stealing values.
- /// It represents the first value being stolen in the batch. `u16` is used
- /// in order to distinguish between `head == tail` and `head == tail -
- /// capacity`.
+ /// Contains two `UnsignedShort` values. The LSB byte is the "real" head of
+ /// the queue. The `UnsignedShort` in the MSB is set by a stealer in process
+ /// of stealing values. It represents the first value being stolen in the
+ /// batch. The `UnsignedShort` indices are intentionally wider than strictly
+ /// required for buffer indexing in order to provide ABA mitigation and make
+ /// it possible to distinguish between full and empty buffers.
///
- /// When both `u16` values are the same, there is no active stealer.
+ /// When both `UnsignedShort` values are the same, there is no active
+ /// stealer.
///
/// Tracking an in-progress stealer prevents a wrapping scenario.
- head: AtomicU32,
+ head: AtomicUnsignedLong,
/// Only updated by producer thread but read by many threads.
- tail: AtomicU16,
+ tail: AtomicUnsignedShort,
/// Elements
buffer: Box<[UnsafeCell<MaybeUninit<task::Notified<T>>>; LOCAL_QUEUE_CAPACITY]>,
@@ -65,7 +82,7 @@
}
/// Create a new local run-queue
-pub(super) fn local<T: 'static>() -> (Steal<T>, Local<T>) {
+pub(crate) fn local<T: 'static>() -> (Steal<T>, Local<T>) {
let mut buffer = Vec::with_capacity(LOCAL_QUEUE_CAPACITY);
for _ in 0..LOCAL_QUEUE_CAPACITY {
@@ -73,8 +90,8 @@
}
let inner = Arc::new(Inner {
- head: AtomicU32::new(0),
- tail: AtomicU16::new(0),
+ head: AtomicUnsignedLong::new(0),
+ tail: AtomicUnsignedShort::new(0),
buffer: make_fixed_size(buffer.into_boxed_slice()),
});
@@ -88,8 +105,8 @@
}
impl<T> Local<T> {
- /// Returns true if the queue has entries that can be stealed.
- pub(super) fn is_stealable(&self) -> bool {
+ /// Returns true if the queue has entries that can be stolen.
+ pub(crate) fn is_stealable(&self) -> bool {
!self.inner.is_empty()
}
@@ -97,12 +114,17 @@
///
/// Separate to is_stealable so that refactors of is_stealable to "protect"
/// some tasks from stealing won't affect this
- pub(super) fn has_tasks(&self) -> bool {
+ pub(crate) fn has_tasks(&self) -> bool {
!self.inner.is_empty()
}
/// Pushes a task to the back of the local queue, skipping the LIFO slot.
- pub(super) fn push_back(&mut self, mut task: task::Notified<T>, inject: &Inject<T>) {
+ pub(crate) fn push_back(
+ &mut self,
+ mut task: task::Notified<T>,
+ inject: &Inject<T>,
+ metrics: &mut MetricsBatch,
+ ) {
let tail = loop {
let head = self.inner.head.load(Acquire);
let (steal, real) = unpack(head);
@@ -110,7 +132,7 @@
// safety: this is the **only** thread that updates this cell.
let tail = unsafe { self.inner.tail.unsync_load() };
- if tail.wrapping_sub(steal) < LOCAL_QUEUE_CAPACITY as u16 {
+ if tail.wrapping_sub(steal) < LOCAL_QUEUE_CAPACITY as UnsignedShort {
// There is capacity for the task
break tail;
} else if steal != real {
@@ -121,7 +143,7 @@
} else {
// Push the current task and half of the queue into the
// inject queue.
- match self.push_overflow(task, real, tail, inject) {
+ match self.push_overflow(task, real, tail, inject, metrics) {
Ok(_) => return,
// Lost the race, try again
Err(v) => {
@@ -160,15 +182,16 @@
fn push_overflow(
&mut self,
task: task::Notified<T>,
- head: u16,
- tail: u16,
+ head: UnsignedShort,
+ tail: UnsignedShort,
inject: &Inject<T>,
+ metrics: &mut MetricsBatch,
) -> Result<(), task::Notified<T>> {
/// How many elements are we taking from the local queue.
///
/// This is one less than the number of tasks pushed to the inject
/// queue as we are also inserting the `task` argument.
- const NUM_TASKS_TAKEN: u16 = (LOCAL_QUEUE_CAPACITY / 2) as u16;
+ const NUM_TASKS_TAKEN: UnsignedShort = (LOCAL_QUEUE_CAPACITY / 2) as UnsignedShort;
assert_eq!(
tail.wrapping_sub(head) as usize,
@@ -213,15 +236,15 @@
/// An iterator that takes elements out of the run queue.
struct BatchTaskIter<'a, T: 'static> {
buffer: &'a [UnsafeCell<MaybeUninit<task::Notified<T>>>; LOCAL_QUEUE_CAPACITY],
- head: u32,
- i: u32,
+ head: UnsignedLong,
+ i: UnsignedLong,
}
impl<'a, T: 'static> Iterator for BatchTaskIter<'a, T> {
type Item = task::Notified<T>;
#[inline]
fn next(&mut self) -> Option<task::Notified<T>> {
- if self.i == u32::from(NUM_TASKS_TAKEN) {
+ if self.i == UnsignedLong::from(NUM_TASKS_TAKEN) {
None
} else {
let i_idx = self.i.wrapping_add(self.head) as usize & MASK;
@@ -240,17 +263,20 @@
// safety: The CAS above ensures that no consumer will look at these
// values again, and we are the only producer.
let batch_iter = BatchTaskIter {
- buffer: &*self.inner.buffer,
- head: head as u32,
+ buffer: &self.inner.buffer,
+ head: head as UnsignedLong,
i: 0,
};
inject.push_batch(batch_iter.chain(std::iter::once(task)));
+ // Add 1 to factor in the task currently being scheduled.
+ metrics.incr_overflow_count();
+
Ok(())
}
/// Pops a task from the local queue.
- pub(super) fn pop(&mut self) -> Option<task::Notified<T>> {
+ pub(crate) fn pop(&mut self) -> Option<task::Notified<T>> {
let mut head = self.inner.head.load(Acquire);
let idx = loop {
@@ -292,15 +318,15 @@
}
impl<T> Steal<T> {
- pub(super) fn is_empty(&self) -> bool {
+ pub(crate) fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Steals half the tasks from self and place them into `dst`.
- pub(super) fn steal_into(
+ pub(crate) fn steal_into(
&self,
dst: &mut Local<T>,
- stats: &mut WorkerStatsBatcher,
+ dst_metrics: &mut MetricsBatch,
) -> Option<task::Notified<T>> {
// Safety: the caller is the only thread that mutates `dst.tail` and
// holds a mutable reference.
@@ -311,7 +337,7 @@
// from `dst` there may not be enough capacity to steal.
let (steal, _) = unpack(dst.inner.head.load(Acquire));
- if dst_tail.wrapping_sub(steal) > LOCAL_QUEUE_CAPACITY as u16 / 2 {
+ if dst_tail.wrapping_sub(steal) > LOCAL_QUEUE_CAPACITY as UnsignedShort / 2 {
// we *could* try to steal less here, but for simplicity, we're just
// going to abort.
return None;
@@ -320,13 +346,15 @@
// Steal the tasks into `dst`'s buffer. This does not yet expose the
// tasks in `dst`.
let mut n = self.steal_into2(dst, dst_tail);
- stats.incr_steal_count(n);
if n == 0 {
// No tasks were stolen
return None;
}
+ dst_metrics.incr_steal_count(n as u16);
+ dst_metrics.incr_steal_operations();
+
// We are returning a task here
n -= 1;
@@ -350,7 +378,7 @@
// Steal tasks from `self`, placing them into `dst`. Returns the number of
// tasks that were stolen.
- fn steal_into2(&self, dst: &mut Local<T>, dst_tail: u16) -> u16 {
+ fn steal_into2(&self, dst: &mut Local<T>, dst_tail: UnsignedShort) -> UnsignedShort {
let mut prev_packed = self.0.head.load(Acquire);
let mut next_packed;
@@ -392,7 +420,11 @@
}
};
- assert!(n <= LOCAL_QUEUE_CAPACITY as u16 / 2, "actual = {}", n);
+ assert!(
+ n <= LOCAL_QUEUE_CAPACITY as UnsignedShort / 2,
+ "actual = {}",
+ n
+ );
let (first, _) = unpack(next_packed);
@@ -446,6 +478,14 @@
}
}
+cfg_metrics! {
+ impl<T> Steal<T> {
+ pub(crate) fn len(&self) -> usize {
+ self.0.len() as _
+ }
+ }
+}
+
impl<T> Clone for Steal<T> {
fn clone(&self) -> Steal<T> {
Steal(self.0.clone())
@@ -461,26 +501,30 @@
}
impl<T> Inner<T> {
- fn is_empty(&self) -> bool {
+ fn len(&self) -> UnsignedShort {
let (_, head) = unpack(self.head.load(Acquire));
let tail = self.tail.load(Acquire);
- head == tail
+ tail.wrapping_sub(head)
+ }
+
+ fn is_empty(&self) -> bool {
+ self.len() == 0
}
}
/// Split the head value into the real head and the index a stealer is working
/// on.
-fn unpack(n: u32) -> (u16, u16) {
- let real = n & u16::MAX as u32;
- let steal = n >> 16;
+fn unpack(n: UnsignedLong) -> (UnsignedShort, UnsignedShort) {
+ let real = n & UnsignedShort::MAX as UnsignedLong;
+ let steal = n >> (mem::size_of::<UnsignedShort>() * 8);
- (steal as u16, real as u16)
+ (steal as UnsignedShort, real as UnsignedShort)
}
/// Join the two head values
-fn pack(steal: u16, real: u16) -> u32 {
- (real as u32) | ((steal as u32) << 16)
+fn pack(steal: UnsignedShort, real: UnsignedShort) -> UnsignedLong {
+ (real as UnsignedLong) | ((steal as UnsignedLong) << (mem::size_of::<UnsignedShort>() * 8))
}
#[test]
diff --git a/src/runtime/thread_pool/worker.rs b/src/runtime/scheduler/multi_thread/worker.rs
similarity index 66%
rename from src/runtime/thread_pool/worker.rs
rename to src/runtime/scheduler/multi_thread/worker.rs
index ae8efe6..148255a 100644
--- a/src/runtime/thread_pool/worker.rs
+++ b/src/runtime/scheduler/multi_thread/worker.rs
@@ -56,27 +56,24 @@
//! the inject queue indefinitely. This would be a ref-count cycle and a memory
//! leak.
-use crate::coop;
-use crate::future::Future;
-use crate::loom::rand::seed;
use crate::loom::sync::{Arc, Mutex};
-use crate::park::{Park, Unpark};
use crate::runtime;
-use crate::runtime::enter::EnterContext;
-use crate::runtime::park::{Parker, Unparker};
-use crate::runtime::stats::{RuntimeStats, WorkerStatsBatcher};
-use crate::runtime::task::{Inject, JoinHandle, OwnedTasks};
-use crate::runtime::thread_pool::{AtomicCell, Idle};
-use crate::runtime::{queue, task, Callback};
-use crate::util::FastRand;
+use crate::runtime::context;
+use crate::runtime::scheduler::multi_thread::{queue, Handle, Idle, Parker, Unparker};
+use crate::runtime::task::{Inject, OwnedTasks};
+use crate::runtime::{
+ blocking, coop, driver, scheduler, task, Config, MetricsBatch, SchedulerMetrics, WorkerMetrics,
+};
+use crate::util::atomic_cell::AtomicCell;
+use crate::util::rand::{FastRand, RngSeedGenerator};
use std::cell::RefCell;
use std::time::Duration;
/// A scheduler worker
pub(super) struct Worker {
- /// Reference to shared state
- shared: Arc<Shared>,
+ /// Reference to scheduler's handle
+ handle: Arc<Handle>,
/// Index holding this worker's remote state
index: usize,
@@ -88,7 +85,7 @@
/// Core data
struct Core {
/// Used to schedule bookkeeping tasks every so often.
- tick: u8,
+ tick: u32,
/// When a task is scheduled from a worker, it is stored in this slot. The
/// worker will check this slot for a task **before** checking the run
@@ -98,7 +95,7 @@
lifo_slot: Option<Notified>,
/// The worker-local run queue.
- run_queue: queue::Local<Arc<Shared>>,
+ run_queue: queue::Local<Arc<Handle>>,
/// True if the worker is currently searching for more work. Searching
/// involves attempting to steal from other workers.
@@ -113,8 +110,8 @@
/// borrow checker happy.
park: Option<Parker>,
- /// Batching stats so they can be submitted to RuntimeStats.
- stats: WorkerStatsBatcher,
+ /// Batching metrics so they can be submitted to RuntimeMetrics.
+ metrics: MetricsBatch,
/// Fast random number generator.
rand: FastRand,
@@ -126,14 +123,16 @@
/// how they communicate between each other.
remotes: Box<[Remote]>,
- /// Submits work to the scheduler while **not** currently on a worker thread.
- inject: Inject<Arc<Shared>>,
+ /// Global task queue used for:
+ /// 1. Submit work to the scheduler while **not** currently on a worker thread.
+ /// 2. Submit work to the scheduler when a worker run queue is saturated
+ inject: Inject<Arc<Handle>>,
/// Coordinates idle workers
idle: Idle,
/// Collection of all active tasks spawned onto this executor.
- owned: OwnedTasks<Arc<Shared>>,
+ pub(super) owned: OwnedTasks<Arc<Handle>>,
/// Cores that have observed the shutdown signal
///
@@ -142,19 +141,19 @@
#[allow(clippy::vec_box)] // we're moving an already-boxed value
shutdown_cores: Mutex<Vec<Box<Core>>>,
- /// Callback for a worker parking itself
- before_park: Option<Callback>,
- /// Callback for a worker unparking itself
- after_unpark: Option<Callback>,
+ /// Scheduler configuration options
+ config: Config,
- /// Collects stats from the runtime.
- stats: RuntimeStats,
+ /// Collects metrics from the runtime.
+ pub(super) scheduler_metrics: SchedulerMetrics,
+
+ pub(super) worker_metrics: Box<[WorkerMetrics]>,
}
/// Used to communicate with a worker from other threads.
struct Remote {
/// Steals tasks from this worker.
- steal: queue::Steal<Arc<Shared>>,
+ steal: queue::Steal<Arc<Handle>>,
/// Unparks the associated worker thread
unpark: Unparker,
@@ -178,10 +177,10 @@
type RunResult = Result<Box<Core>, ()>;
/// A task handle
-type Task = task::Task<Arc<Shared>>;
+type Task = task::Task<Arc<Handle>>;
/// A notified task handle
-type Notified = task::Notified<Arc<Shared>>;
+type Notified = task::Notified<Arc<Handle>>;
// Tracks thread-local state
scoped_thread_local!(static CURRENT: Context);
@@ -189,14 +188,17 @@
pub(super) fn create(
size: usize,
park: Parker,
- before_park: Option<Callback>,
- after_unpark: Option<Callback>,
-) -> (Arc<Shared>, Launch) {
- let mut cores = vec![];
- let mut remotes = vec![];
+ driver_handle: driver::Handle,
+ blocking_spawner: blocking::Spawner,
+ seed_generator: RngSeedGenerator,
+ config: Config,
+) -> (Arc<Handle>, Launch) {
+ let mut cores = Vec::with_capacity(size);
+ let mut remotes = Vec::with_capacity(size);
+ let mut worker_metrics = Vec::with_capacity(size);
// Create the local queues
- for i in 0..size {
+ for _ in 0..size {
let (steal, run_queue) = queue::local();
let park = park.clone();
@@ -209,37 +211,44 @@
is_searching: false,
is_shutdown: false,
park: Some(park),
- stats: WorkerStatsBatcher::new(i),
- rand: FastRand::new(seed()),
+ metrics: MetricsBatch::new(),
+ rand: FastRand::new(config.seed_generator.next_seed()),
}));
remotes.push(Remote { steal, unpark });
+ worker_metrics.push(WorkerMetrics::new());
}
- let shared = Arc::new(Shared {
- remotes: remotes.into_boxed_slice(),
- inject: Inject::new(),
- idle: Idle::new(size),
- owned: OwnedTasks::new(),
- shutdown_cores: Mutex::new(vec![]),
- before_park,
- after_unpark,
- stats: RuntimeStats::new(size),
+ let handle = Arc::new(Handle {
+ shared: Shared {
+ remotes: remotes.into_boxed_slice(),
+ inject: Inject::new(),
+ idle: Idle::new(size),
+ owned: OwnedTasks::new(),
+ shutdown_cores: Mutex::new(vec![]),
+ config,
+ scheduler_metrics: SchedulerMetrics::new(),
+ worker_metrics: worker_metrics.into_boxed_slice(),
+ },
+ driver: driver_handle,
+ blocking_spawner,
+ seed_generator,
});
let mut launch = Launch(vec![]);
for (index, core) in cores.drain(..).enumerate() {
launch.0.push(Arc::new(Worker {
- shared: shared.clone(),
+ handle: handle.clone(),
index,
core: AtomicCell::new(Some(core)),
}));
}
- (shared, launch)
+ (handle, launch)
}
+#[track_caller]
pub(crate) fn block_in_place<F, R>(f: F) -> R
where
F: FnOnce() -> R,
@@ -266,35 +275,45 @@
let mut had_entered = false;
- CURRENT.with(|maybe_cx| {
- match (crate::runtime::enter::context(), maybe_cx.is_some()) {
- (EnterContext::Entered { .. }, true) => {
+ let setup_result = CURRENT.with(|maybe_cx| {
+ match (
+ crate::runtime::context::current_enter_context(),
+ maybe_cx.is_some(),
+ ) {
+ (context::EnterRuntime::Entered { .. }, true) => {
// We are on a thread pool runtime thread, so we just need to
// set up blocking.
had_entered = true;
}
- (EnterContext::Entered { allow_blocking }, false) => {
+ (
+ context::EnterRuntime::Entered {
+ allow_block_in_place,
+ },
+ false,
+ ) => {
// We are on an executor, but _not_ on the thread pool. That is
// _only_ okay if we are in a thread pool runtime's block_on
// method:
- if allow_blocking {
+ if allow_block_in_place {
had_entered = true;
- return;
+ return Ok(());
} else {
- // This probably means we are on the basic_scheduler or in a
+ // This probably means we are on the current_thread runtime or in a
// LocalSet, where it is _not_ okay to block.
- panic!("can call blocking only when running on the multi-threaded runtime");
+ return Err(
+ "can call blocking only when running on the multi-threaded runtime",
+ );
}
}
- (EnterContext::NotEntered, true) => {
+ (context::EnterRuntime::NotEntered, true) => {
// This is a nested call to block_in_place (we already exited).
// All the necessary setup has already been done.
- return;
+ return Ok(());
}
- (EnterContext::NotEntered, false) => {
+ (context::EnterRuntime::NotEntered, false) => {
// We are outside of the tokio runtime, so blocking is fine.
// We can also skip all of the thread pool blocking setup steps.
- return;
+ return Ok(());
}
}
@@ -303,7 +322,7 @@
// Get the worker core. If none is set, then blocking is fine!
let core = match cx.core.borrow_mut().take() {
Some(core) => core,
- None => return,
+ None => return Ok(()),
};
// The parker should be set here
@@ -322,25 +341,24 @@
// steal the core back.
let worker = cx.worker.clone();
runtime::spawn_blocking(move || run(worker));
+ Ok(())
});
+ if let Err(panic_message) = setup_result {
+ panic!("{}", panic_message);
+ }
+
if had_entered {
// Unset the current task's budget. Blocking sections are not
// constrained by task budgets.
let _reset = Reset(coop::stop());
- crate::runtime::enter::exit(f)
+ crate::runtime::context::exit_runtime(f)
} else {
f()
}
}
-/// After how many ticks is the global queue polled. This helps to ensure
-/// fairness.
-///
-/// The number is fairly arbitrary. I believe this value was copied from golang.
-const GLOBAL_POLL_INTERVAL: u8 = 61;
-
impl Launch {
pub(crate) fn launch(mut self) {
for worker in self.0.drain(..) {
@@ -350,6 +368,22 @@
}
fn run(worker: Arc<Worker>) {
+ struct AbortOnPanic;
+
+ impl Drop for AbortOnPanic {
+ fn drop(&mut self) {
+ if std::thread::panicking() {
+ eprintln!("worker thread panicking; aborting process");
+ std::process::abort();
+ }
+ }
+ }
+
+ // Catching panics on worker threads in tests is quite tricky. Instead, when
+ // debug assertions are enabled, we just abort the process.
+ #[cfg(debug_assertions)]
+ let _abort_on_panic = AbortOnPanic;
+
// Acquire a core. If this fails, then another thread is running this
// worker and there is nothing further to do.
let core = match worker.core.take() {
@@ -357,18 +391,24 @@
None => return,
};
+ let handle = scheduler::Handle::MultiThread(worker.handle.clone());
+ let _enter = crate::runtime::context::enter_runtime(&handle, true);
+
// Set the worker context.
let cx = Context {
worker,
core: RefCell::new(None),
};
- let _enter = crate::runtime::enter(true);
-
CURRENT.set(&cx, || {
// This should always be an error. It only returns a `Result` to support
// using `?` to short circuit.
assert!(cx.run(core).is_err());
+
+ // Check if there are any deferred tasks to notify. This can happen when
+ // the worker core is lost due to `block_in_place()` being called from
+ // within the task.
+ wake_deferred_tasks();
});
}
@@ -393,26 +433,30 @@
core = self.run_task(task, core)?;
} else {
// Wait for work
- core = self.park(core);
+ core = if did_defer_tasks() {
+ self.park_timeout(core, Some(Duration::from_millis(0)))
+ } else {
+ self.park(core)
+ };
}
}
core.pre_shutdown(&self.worker);
// Signal shutdown
- self.worker.shared.shutdown(core);
+ self.worker.handle.shutdown_core(core);
Err(())
}
fn run_task(&self, task: Notified, mut core: Box<Core>) -> RunResult {
- let task = self.worker.shared.owned.assert_owner(task);
+ let task = self.worker.handle.shared.owned.assert_owner(task);
// Make sure the worker is not in the **searching** state. This enables
// another idle worker to try to steal work.
core.transition_from_searching(&self.worker);
// Make the core available to the runtime context
- core.stats.incr_poll_count();
+ core.metrics.incr_poll_count();
*self.core.borrow_mut() = Some(core);
// Run the task
@@ -437,14 +481,15 @@
if coop::has_budget_remaining() {
// Run the LIFO task, then loop
- core.stats.incr_poll_count();
+ core.metrics.incr_poll_count();
*self.core.borrow_mut() = Some(core);
- let task = self.worker.shared.owned.assert_owner(task);
+ let task = self.worker.handle.shared.owned.assert_owner(task);
task.run();
} else {
// Not enough budget left to run the LIFO task, push it to
// the back of the queue and return.
- core.run_queue.push_back(task, self.worker.inject());
+ core.run_queue
+ .push_back(task, self.worker.inject(), &mut core.metrics);
return Ok(core);
}
}
@@ -452,7 +497,7 @@
}
fn maintenance(&self, mut core: Box<Core>) -> Box<Core> {
- if core.tick % GLOBAL_POLL_INTERVAL == 0 {
+ if core.tick % self.worker.handle.shared.config.event_interval == 0 {
// Call `park` with a 0 timeout. This enables the I/O driver, timer, ...
// to run without actually putting the thread to sleep.
core = self.park_timeout(core, Some(Duration::from_millis(0)));
@@ -464,14 +509,27 @@
core
}
+ /// Parks the worker thread while waiting for tasks to execute.
+ ///
+ /// This function checks if indeed there's no more work left to be done before parking.
+ /// Also important to notice that, before parking, the worker thread will try to take
+ /// ownership of the Driver (IO/Time) and dispatch any events that might have fired.
+ /// Whenever a worker thread executes the Driver loop, all waken tasks are scheduled
+ /// in its own local queue until the queue saturates (ntasks > LOCAL_QUEUE_CAPACITY).
+ /// When the local queue is saturated, the overflow tasks are added to the injection queue
+ /// from where other workers can pick them up.
+ /// Also, we rely on the workstealing algorithm to spread the tasks amongst workers
+ /// after all the IOs get dispatched
fn park(&self, mut core: Box<Core>) -> Box<Core> {
- if let Some(f) = &self.worker.shared.before_park {
+ if let Some(f) = &self.worker.handle.shared.config.before_park {
f();
}
if core.transition_to_parked(&self.worker) {
while !core.is_shutdown {
+ core.metrics.about_to_park();
core = self.park_timeout(core, None);
+ core.metrics.returned_from_park();
// Run regularly scheduled maintenance
core.maintenance(&self.worker);
@@ -482,7 +540,7 @@
}
}
- if let Some(f) = &self.worker.shared.after_unpark {
+ if let Some(f) = &self.worker.handle.shared.config.after_unpark {
f();
}
core
@@ -492,31 +550,30 @@
// Take the parker out of core
let mut park = core.park.take().expect("park missing");
- core.stats.about_to_park();
-
// Store `core` in context
*self.core.borrow_mut() = Some(core);
// Park thread
if let Some(timeout) = duration {
- park.park_timeout(timeout).expect("park failed");
+ park.park_timeout(&self.worker.handle.driver, timeout);
} else {
- park.park().expect("park failed");
+ park.park(&self.worker.handle.driver);
}
+ wake_deferred_tasks();
+
// Remove `core` from context
core = self.core.borrow_mut().take().expect("core missing");
// Place `park` back in `core`
core.park = Some(park);
- // If there are tasks available to steal, notify a worker
- if core.run_queue.is_stealable() {
- self.worker.shared.notify_parked();
+ // If there are tasks available to steal, but this worker is not
+ // looking for tasks to steal, notify another worker.
+ if !core.is_searching && core.run_queue.is_stealable() {
+ self.worker.handle.notify_parked();
}
- core.stats.returned_from_park();
-
core
}
}
@@ -529,7 +586,7 @@
/// Return the next notified task available to this worker.
fn next_task(&mut self, worker: &Worker) -> Option<Notified> {
- if self.tick % GLOBAL_POLL_INTERVAL == 0 {
+ if self.tick % worker.handle.shared.config.global_queue_interval == 0 {
worker.inject().pop().or_else(|| self.next_local_task())
} else {
self.next_local_task().or_else(|| worker.inject().pop())
@@ -540,12 +597,17 @@
self.lifo_slot.take().or_else(|| self.run_queue.pop())
}
+ /// Function responsible for stealing tasks from another worker
+ ///
+ /// Note: Only if less than half the workers are searching for tasks to steal
+ /// a new worker will actually try to steal. The idea is to make sure not all
+ /// workers will be trying to steal at the same time.
fn steal_work(&mut self, worker: &Worker) -> Option<Notified> {
if !self.transition_to_searching(worker) {
return None;
}
- let num = worker.shared.remotes.len();
+ let num = worker.handle.shared.remotes.len();
// Start from a random worker
let start = self.rand.fastrand_n(num as u32) as usize;
@@ -557,22 +619,22 @@
continue;
}
- let target = &worker.shared.remotes[i];
+ let target = &worker.handle.shared.remotes[i];
if let Some(task) = target
.steal
- .steal_into(&mut self.run_queue, &mut self.stats)
+ .steal_into(&mut self.run_queue, &mut self.metrics)
{
return Some(task);
}
}
// Fallback on checking the global queue
- worker.shared.inject.pop()
+ worker.handle.shared.inject.pop()
}
fn transition_to_searching(&mut self, worker: &Worker) -> bool {
if !self.is_searching {
- self.is_searching = worker.shared.idle.transition_worker_to_searching();
+ self.is_searching = worker.handle.shared.idle.transition_worker_to_searching();
}
self.is_searching
@@ -584,12 +646,12 @@
}
self.is_searching = false;
- worker.shared.transition_worker_from_searching();
+ worker.handle.transition_worker_from_searching();
}
/// Prepares the worker state for parking.
///
- /// Returns true if the transition happend, false if there is work to do first.
+ /// Returns true if the transition happened, false if there is work to do first.
fn transition_to_parked(&mut self, worker: &Worker) -> bool {
// Workers should not park if they have work to do
if self.lifo_slot.is_some() || self.run_queue.has_tasks() {
@@ -600,6 +662,7 @@
// must check all the queues one last time in case work materialized
// between the last work scan and transitioning out of searching.
let is_last_searcher = worker
+ .handle
.shared
.idle
.transition_worker_to_parked(worker.index, self.is_searching);
@@ -609,7 +672,7 @@
self.is_searching = false;
if is_last_searcher {
- worker.shared.notify_if_work_pending();
+ worker.handle.notify_if_work_pending();
}
true
@@ -620,12 +683,15 @@
// If a task is in the lifo slot, then we must unpark regardless of
// being notified
if self.lifo_slot.is_some() {
- worker.shared.idle.unpark_worker_by_id(worker.index);
- self.is_searching = true;
+ // When a worker wakes, it should only transition to the "searching"
+ // state when the wake originates from another worker *or* a new task
+ // is pushed. We do *not* want the worker to transition to "searching"
+ // when it wakes when the I/O driver receives new events.
+ self.is_searching = !worker.handle.shared.idle.unpark_worker_by_id(worker.index);
return true;
}
- if worker.shared.idle.is_parked(worker.index) {
+ if worker.handle.shared.idle.is_parked(worker.index) {
return false;
}
@@ -636,7 +702,8 @@
/// Runs maintenance work such as checking the pool's state.
fn maintenance(&mut self, worker: &Worker) {
- self.stats.submit(&worker.shared.stats);
+ self.metrics
+ .submit(&worker.handle.shared.worker_metrics[worker.index]);
if !self.is_shutdown {
// Check if the scheduler has been shutdown
@@ -648,68 +715,52 @@
/// before we enter the single-threaded phase of shutdown processing.
fn pre_shutdown(&mut self, worker: &Worker) {
// Signal to all tasks to shut down.
- worker.shared.owned.close_and_shutdown_all();
+ worker.handle.shared.owned.close_and_shutdown_all();
- self.stats.submit(&worker.shared.stats);
+ self.metrics
+ .submit(&worker.handle.shared.worker_metrics[worker.index]);
}
/// Shuts down the core.
- fn shutdown(&mut self) {
+ fn shutdown(&mut self, handle: &Handle) {
// Take the core
let mut park = self.park.take().expect("park missing");
// Drain the queue
while self.next_local_task().is_some() {}
- park.shutdown();
+ park.shutdown(&handle.driver);
}
}
impl Worker {
/// Returns a reference to the scheduler's injection queue.
- fn inject(&self) -> &Inject<Arc<Shared>> {
- &self.shared.inject
+ fn inject(&self) -> &Inject<Arc<Handle>> {
+ &self.handle.shared.inject
}
}
-impl task::Schedule for Arc<Shared> {
+// TODO: Move `Handle` impls into handle.rs
+impl task::Schedule for Arc<Handle> {
fn release(&self, task: &Task) -> Option<Task> {
- self.owned.remove(task)
+ self.shared.owned.remove(task)
}
fn schedule(&self, task: Notified) {
- (**self).schedule(task, false);
+ self.schedule_task(task, false);
}
fn yield_now(&self, task: Notified) {
- (**self).schedule(task, true);
+ self.schedule_task(task, true);
}
}
-impl Shared {
- pub(super) fn bind_new_task<T>(me: &Arc<Self>, future: T) -> JoinHandle<T::Output>
- where
- T: Future + Send + 'static,
- T::Output: Send + 'static,
- {
- let (handle, notified) = me.owned.bind(future, me.clone());
-
- if let Some(notified) = notified {
- me.schedule(notified, false);
- }
-
- handle
- }
-
- pub(crate) fn stats(&self) -> &RuntimeStats {
- &self.stats
- }
-
- pub(super) fn schedule(&self, task: Notified, is_yield: bool) {
+impl Handle {
+ pub(super) fn schedule_task(&self, task: Notified, is_yield: bool) {
CURRENT.with(|maybe_cx| {
if let Some(cx) = maybe_cx {
// Make sure the task is part of the **current** scheduler.
- if self.ptr_eq(&cx.worker.shared) {
+ if self.ptr_eq(&cx.worker.handle) {
// And the current thread still holds a core
if let Some(core) = cx.core.borrow_mut().as_mut() {
self.schedule_local(core, task, is_yield);
@@ -719,18 +770,22 @@
}
// Otherwise, use the inject queue.
- self.inject.push(task);
+ self.shared.inject.push(task);
+ self.shared.scheduler_metrics.inc_remote_schedule_count();
self.notify_parked();
})
}
fn schedule_local(&self, core: &mut Core, task: Notified, is_yield: bool) {
+ core.metrics.inc_local_schedule_count();
+
// Spawning from the worker thread. If scheduling a "yield" then the
// task must always be pushed to the back of the queue, enabling other
// tasks to be executed. If **not** a yield, then there is more
// flexibility and the task may go to the front of the queue.
- let should_notify = if is_yield {
- core.run_queue.push_back(task, &self.inject);
+ let should_notify = if is_yield || self.shared.config.disable_lifo_slot {
+ core.run_queue
+ .push_back(task, &self.shared.inject, &mut core.metrics);
true
} else {
// Push to the LIFO slot
@@ -738,7 +793,8 @@
let ret = prev.is_some();
if let Some(prev) = prev {
- core.run_queue.push_back(prev, &self.inject);
+ core.run_queue
+ .push_back(prev, &self.shared.inject, &mut core.metrics);
}
core.lifo_slot = Some(task);
@@ -755,38 +811,38 @@
}
pub(super) fn close(&self) {
- if self.inject.close() {
+ if self.shared.inject.close() {
self.notify_all();
}
}
fn notify_parked(&self) {
- if let Some(index) = self.idle.worker_to_notify() {
- self.remotes[index].unpark.unpark();
+ if let Some(index) = self.shared.idle.worker_to_notify() {
+ self.shared.remotes[index].unpark.unpark(&self.driver);
}
}
fn notify_all(&self) {
- for remote in &self.remotes[..] {
- remote.unpark.unpark();
+ for remote in &self.shared.remotes[..] {
+ remote.unpark.unpark(&self.driver);
}
}
fn notify_if_work_pending(&self) {
- for remote in &self.remotes[..] {
+ for remote in &self.shared.remotes[..] {
if !remote.steal.is_empty() {
self.notify_parked();
return;
}
}
- if !self.inject.is_empty() {
+ if !self.shared.inject.is_empty() {
self.notify_parked();
}
}
fn transition_worker_from_searching(&self) {
- if self.idle.transition_worker_from_searching() {
+ if self.shared.idle.transition_worker_from_searching() {
// We are the final searching worker. Because work was found, we
// need to notify another worker.
self.notify_parked();
@@ -797,29 +853,49 @@
/// its core back into its handle.
///
/// If all workers have reached this point, the final cleanup is performed.
- fn shutdown(&self, core: Box<Core>) {
- let mut cores = self.shutdown_cores.lock();
+ fn shutdown_core(&self, core: Box<Core>) {
+ let mut cores = self.shared.shutdown_cores.lock();
cores.push(core);
- if cores.len() != self.remotes.len() {
+ if cores.len() != self.shared.remotes.len() {
return;
}
- debug_assert!(self.owned.is_empty());
+ debug_assert!(self.shared.owned.is_empty());
for mut core in cores.drain(..) {
- core.shutdown();
+ core.shutdown(self);
}
// Drain the injection queue
//
// We already shut down every task, so we can simply drop the tasks.
- while let Some(task) = self.inject.pop() {
+ while let Some(task) = self.shared.inject.pop() {
drop(task);
}
}
- fn ptr_eq(&self, other: &Shared) -> bool {
+ fn ptr_eq(&self, other: &Handle) -> bool {
std::ptr::eq(self, other)
}
}
+
+fn did_defer_tasks() -> bool {
+ context::with_defer(|deferred| !deferred.is_empty()).unwrap()
+}
+
+fn wake_deferred_tasks() {
+ context::with_defer(|deferred| deferred.wake());
+}
+
+cfg_metrics! {
+ impl Shared {
+ pub(super) fn injection_queue_depth(&self) -> usize {
+ self.inject.len()
+ }
+
+ pub(super) fn worker_local_queue_depth(&self, worker: usize) -> usize {
+ self.remotes[worker].steal.len()
+ }
+ }
+}
diff --git a/src/runtime/signal/mod.rs b/src/runtime/signal/mod.rs
new file mode 100644
index 0000000..24f2f4c
--- /dev/null
+++ b/src/runtime/signal/mod.rs
@@ -0,0 +1,142 @@
+#![cfg_attr(not(feature = "rt"), allow(dead_code))]
+
+//! Signal driver
+
+use crate::runtime::{driver, io};
+use crate::signal::registry::globals;
+
+use mio::net::UnixStream;
+use std::io::{self as std_io, Read};
+use std::sync::{Arc, Weak};
+use std::time::Duration;
+
+/// Responsible for registering wakeups when an OS signal is received, and
+/// subsequently dispatching notifications to any signal listeners as appropriate.
+///
+/// Note: this driver relies on having an enabled IO driver in order to listen to
+/// pipe write wakeups.
+#[derive(Debug)]
+pub(crate) struct Driver {
+ /// Thread parker. The `Driver` park implementation delegates to this.
+ io: io::Driver,
+
+ /// A pipe for receiving wake events from the signal handler
+ receiver: UnixStream,
+
+ /// Shared state. The driver keeps a strong ref and the handle keeps a weak
+ /// ref. The weak ref is used to check if the driver is still active before
+ /// trying to register a signal handler.
+ inner: Arc<()>,
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct Handle {
+ /// Paired w/ the `Arc` above and is used to check if the driver is still
+ /// around before attempting to register a signal handler.
+ inner: Weak<()>,
+}
+
+// ===== impl Driver =====
+
+impl Driver {
+ /// Creates a new signal `Driver` instance that delegates wakeups to `park`.
+ pub(crate) fn new(io: io::Driver, io_handle: &io::Handle) -> std_io::Result<Self> {
+ use std::mem::ManuallyDrop;
+ use std::os::unix::io::{AsRawFd, FromRawFd};
+
+ // NB: We give each driver a "fresh" receiver file descriptor to avoid
+ // the issues described in alexcrichton/tokio-process#42.
+ //
+ // In the past we would reuse the actual receiver file descriptor and
+ // swallow any errors around double registration of the same descriptor.
+ // I'm not sure if the second (failed) registration simply doesn't end
+ // up receiving wake up notifications, or there could be some race
+ // condition when consuming readiness events, but having distinct
+ // descriptors appears to mitigate this.
+ //
+ // Unfortunately we cannot just use a single global UnixStream instance
+ // either, since we can't assume they will always be registered with the
+ // exact same reactor.
+ //
+ // Mio 0.7 removed `try_clone()` as an API due to unexpected behavior
+ // with registering dups with the same reactor. In this case, duping is
+ // safe as each dup is registered with separate reactors **and** we
+ // only expect at least one dup to receive the notification.
+
+ // Manually drop as we don't actually own this instance of UnixStream.
+ let receiver_fd = globals().receiver.as_raw_fd();
+
+ // safety: there is nothing unsafe about this, but the `from_raw_fd` fn is marked as unsafe.
+ let original =
+ ManuallyDrop::new(unsafe { std::os::unix::net::UnixStream::from_raw_fd(receiver_fd) });
+ let mut receiver = UnixStream::from_std(original.try_clone()?);
+
+ io_handle.register_signal_receiver(&mut receiver)?;
+
+ Ok(Self {
+ io,
+ receiver,
+ inner: Arc::new(()),
+ })
+ }
+
+ /// Returns a handle to this event loop which can be sent across threads
+ /// and can be used as a proxy to the event loop itself.
+ pub(crate) fn handle(&self) -> Handle {
+ Handle {
+ inner: Arc::downgrade(&self.inner),
+ }
+ }
+
+ pub(crate) fn park(&mut self, handle: &driver::Handle) {
+ self.io.park(handle);
+ self.process();
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) {
+ self.io.park_timeout(handle, duration);
+ self.process();
+ }
+
+ pub(crate) fn shutdown(&mut self, handle: &driver::Handle) {
+ self.io.shutdown(handle)
+ }
+
+ fn process(&mut self) {
+ // If the signal pipe has not received a readiness event, then there is
+ // nothing else to do.
+ if !self.io.consume_signal_ready() {
+ return;
+ }
+
+ // Drain the pipe completely so we can receive a new readiness event
+ // if another signal has come in.
+ let mut buf = [0; 128];
+ loop {
+ match self.receiver.read(&mut buf) {
+ Ok(0) => panic!("EOF on self-pipe"),
+ Ok(_) => continue, // Keep reading
+ Err(e) if e.kind() == std_io::ErrorKind::WouldBlock => break,
+ Err(e) => panic!("Bad read on self-pipe: {}", e),
+ }
+ }
+
+ // Broadcast any signals which were received
+ globals().broadcast();
+ }
+}
+
+// ===== impl Handle =====
+
+impl Handle {
+ pub(crate) fn check_inner(&self) -> std_io::Result<()> {
+ if self.inner.strong_count() > 0 {
+ Ok(())
+ } else {
+ Err(std_io::Error::new(
+ std_io::ErrorKind::Other,
+ "signal driver gone",
+ ))
+ }
+ }
+}
diff --git a/src/runtime/spawner.rs b/src/runtime/spawner.rs
deleted file mode 100644
index 9a3d465..0000000
--- a/src/runtime/spawner.rs
+++ /dev/null
@@ -1,47 +0,0 @@
-use crate::future::Future;
-use crate::runtime::basic_scheduler;
-use crate::runtime::stats::RuntimeStats;
-use crate::task::JoinHandle;
-
-cfg_rt_multi_thread! {
- use crate::runtime::thread_pool;
-}
-
-#[derive(Debug, Clone)]
-pub(crate) enum Spawner {
- Basic(basic_scheduler::Spawner),
- #[cfg(feature = "rt-multi-thread")]
- ThreadPool(thread_pool::Spawner),
-}
-
-impl Spawner {
- pub(crate) fn shutdown(&mut self) {
- #[cfg(feature = "rt-multi-thread")]
- {
- if let Spawner::ThreadPool(spawner) = self {
- spawner.shutdown();
- }
- }
- }
-
- pub(crate) fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
- where
- F: Future + Send + 'static,
- F::Output: Send + 'static,
- {
- match self {
- Spawner::Basic(spawner) => spawner.spawn(future),
- #[cfg(feature = "rt-multi-thread")]
- Spawner::ThreadPool(spawner) => spawner.spawn(future),
- }
- }
-
- #[cfg_attr(not(all(tokio_unstable, feature = "stats")), allow(dead_code))]
- pub(crate) fn stats(&self) -> &RuntimeStats {
- match self {
- Spawner::Basic(spawner) => spawner.stats(),
- #[cfg(feature = "rt-multi-thread")]
- Spawner::ThreadPool(spawner) => spawner.stats(),
- }
- }
-}
diff --git a/src/runtime/stats/mock.rs b/src/runtime/stats/mock.rs
deleted file mode 100644
index 3bda8bf..0000000
--- a/src/runtime/stats/mock.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//! This file contains mocks of the types in src/runtime/stats/stats.rs
-
-pub(crate) struct RuntimeStats {}
-
-impl RuntimeStats {
- pub(crate) fn new(_worker_threads: usize) -> Self {
- Self {}
- }
-}
-
-pub(crate) struct WorkerStatsBatcher {}
-
-impl WorkerStatsBatcher {
- pub(crate) fn new(_my_index: usize) -> Self {
- Self {}
- }
-
- pub(crate) fn submit(&mut self, _to: &RuntimeStats) {}
-
- pub(crate) fn about_to_park(&mut self) {}
- pub(crate) fn returned_from_park(&mut self) {}
-
- #[cfg(feature = "rt-multi-thread")]
- pub(crate) fn incr_steal_count(&mut self, _by: u16) {}
-
- pub(crate) fn incr_poll_count(&mut self) {}
-}
diff --git a/src/runtime/stats/mod.rs b/src/runtime/stats/mod.rs
deleted file mode 100644
index 5e08e8e..0000000
--- a/src/runtime/stats/mod.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-//! This module contains information need to view information about how the
-//! runtime is performing.
-#![allow(clippy::module_inception)]
-
-cfg_stats! {
- mod stats;
-
- pub use self::stats::{RuntimeStats, WorkerStats};
- pub(crate) use self::stats::WorkerStatsBatcher;
-}
-
-cfg_not_stats! {
- #[path = "mock.rs"]
- mod stats;
-
- pub(crate) use self::stats::{RuntimeStats, WorkerStatsBatcher};
-}
diff --git a/src/runtime/stats/stats.rs b/src/runtime/stats/stats.rs
deleted file mode 100644
index b2bcacc..0000000
--- a/src/runtime/stats/stats.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-//! This file contains the types necessary to collect various types of stats.
-use crate::loom::sync::atomic::{AtomicU64, Ordering::Relaxed};
-
-use std::convert::TryFrom;
-use std::time::{Duration, Instant};
-
-/// This type contains methods to retrieve stats from a Tokio runtime.
-#[derive(Debug)]
-pub struct RuntimeStats {
- workers: Box<[WorkerStats]>,
-}
-
-/// This type contains methods to retrieve stats from a worker thread on a Tokio runtime.
-#[derive(Debug)]
-#[repr(align(128))]
-pub struct WorkerStats {
- park_count: AtomicU64,
- steal_count: AtomicU64,
- poll_count: AtomicU64,
- busy_duration_total: AtomicU64,
-}
-
-impl RuntimeStats {
- pub(crate) fn new(worker_threads: usize) -> Self {
- let mut workers = Vec::with_capacity(worker_threads);
- for _ in 0..worker_threads {
- workers.push(WorkerStats {
- park_count: AtomicU64::new(0),
- steal_count: AtomicU64::new(0),
- poll_count: AtomicU64::new(0),
- busy_duration_total: AtomicU64::new(0),
- });
- }
-
- Self {
- workers: workers.into_boxed_slice(),
- }
- }
-
- /// Returns a slice containing the worker stats for each worker thread.
- pub fn workers(&self) -> impl Iterator<Item = &WorkerStats> {
- self.workers.iter()
- }
-}
-
-impl WorkerStats {
- /// Returns the total number of times this worker thread has parked.
- pub fn park_count(&self) -> u64 {
- self.park_count.load(Relaxed)
- }
-
- /// Returns the number of tasks this worker has stolen from other worker
- /// threads.
- pub fn steal_count(&self) -> u64 {
- self.steal_count.load(Relaxed)
- }
-
- /// Returns the number of times this worker has polled a task.
- pub fn poll_count(&self) -> u64 {
- self.poll_count.load(Relaxed)
- }
-
- /// Returns the total amount of time this worker has been busy for.
- pub fn total_busy_duration(&self) -> Duration {
- Duration::from_nanos(self.busy_duration_total.load(Relaxed))
- }
-}
-
-pub(crate) struct WorkerStatsBatcher {
- my_index: usize,
- park_count: u64,
- steal_count: u64,
- poll_count: u64,
- /// The total busy duration in nanoseconds.
- busy_duration_total: u64,
- last_resume_time: Instant,
-}
-
-impl WorkerStatsBatcher {
- pub(crate) fn new(my_index: usize) -> Self {
- Self {
- my_index,
- park_count: 0,
- steal_count: 0,
- poll_count: 0,
- busy_duration_total: 0,
- last_resume_time: Instant::now(),
- }
- }
- pub(crate) fn submit(&mut self, to: &RuntimeStats) {
- let worker = &to.workers[self.my_index];
-
- worker.park_count.store(self.park_count, Relaxed);
- worker.steal_count.store(self.steal_count, Relaxed);
- worker.poll_count.store(self.poll_count, Relaxed);
-
- worker
- .busy_duration_total
- .store(self.busy_duration_total, Relaxed);
- }
-
- pub(crate) fn about_to_park(&mut self) {
- self.park_count += 1;
-
- let busy_duration = self.last_resume_time.elapsed();
- let busy_duration = u64::try_from(busy_duration.as_nanos()).unwrap_or(u64::MAX);
- self.busy_duration_total += busy_duration;
- }
-
- pub(crate) fn returned_from_park(&mut self) {
- self.last_resume_time = Instant::now();
- }
-
- #[cfg(feature = "rt-multi-thread")]
- pub(crate) fn incr_steal_count(&mut self, by: u16) {
- self.steal_count += u64::from(by);
- }
-
- pub(crate) fn incr_poll_count(&mut self) {
- self.poll_count += 1;
- }
-}
diff --git a/src/runtime/task/abort.rs b/src/runtime/task/abort.rs
new file mode 100644
index 0000000..6edca10
--- /dev/null
+++ b/src/runtime/task/abort.rs
@@ -0,0 +1,87 @@
+use crate::runtime::task::{Header, RawTask};
+use std::fmt;
+use std::panic::{RefUnwindSafe, UnwindSafe};
+
+/// An owned permission to abort a spawned task, without awaiting its completion.
+///
+/// Unlike a [`JoinHandle`], an `AbortHandle` does *not* represent the
+/// permission to await the task's completion, only to terminate it.
+///
+/// The task may be aborted by calling the [`AbortHandle::abort`] method.
+/// Dropping an `AbortHandle` releases the permission to terminate the task
+/// --- it does *not* abort the task.
+///
+/// [`JoinHandle`]: crate::task::JoinHandle
+#[cfg_attr(docsrs, doc(cfg(feature = "rt")))]
+pub struct AbortHandle {
+ raw: RawTask,
+}
+
+impl AbortHandle {
+ pub(super) fn new(raw: RawTask) -> Self {
+ Self { raw }
+ }
+
+ /// Abort the task associated with the handle.
+ ///
+ /// Awaiting a cancelled task might complete as usual if the task was
+ /// already completed at the time it was cancelled, but most likely it
+ /// will fail with a [cancelled] `JoinError`.
+ ///
+ /// If the task was already cancelled, such as by [`JoinHandle::abort`],
+ /// this method will do nothing.
+ ///
+ /// [cancelled]: method@super::error::JoinError::is_cancelled
+ /// [`JoinHandle::abort`]: method@super::JoinHandle::abort
+ pub fn abort(&self) {
+ self.raw.remote_abort();
+ }
+
+ /// Checks if the task associated with this `AbortHandle` has finished.
+ ///
+ /// Please note that this method can return `false` even if `abort` has been
+ /// called on the task. This is because the cancellation process may take
+ /// some time, and this method does not return `true` until it has
+ /// completed.
+ pub fn is_finished(&self) -> bool {
+ let state = self.raw.state().load();
+ state.is_complete()
+ }
+
+ /// Returns a [task ID] that uniquely identifies this task relative to other
+ /// currently spawned tasks.
+ ///
+ /// **Note**: This is an [unstable API][unstable]. The public API of this type
+ /// may break in 1.x releases. See [the documentation on unstable
+ /// features][unstable] for details.
+ ///
+ /// [task ID]: crate::task::Id
+ /// [unstable]: crate#unstable-features
+ #[cfg(tokio_unstable)]
+ #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
+ pub fn id(&self) -> super::Id {
+ // Safety: The header pointer is valid.
+ unsafe { Header::get_id(self.raw.header_ptr()) }
+ }
+}
+
+unsafe impl Send for AbortHandle {}
+unsafe impl Sync for AbortHandle {}
+
+impl UnwindSafe for AbortHandle {}
+impl RefUnwindSafe for AbortHandle {}
+
+impl fmt::Debug for AbortHandle {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ // Safety: The header pointer is valid.
+ let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) };
+ let id = unsafe { id_ptr.as_ref() };
+ fmt.debug_struct("AbortHandle").field("id", id).finish()
+ }
+}
+
+impl Drop for AbortHandle {
+ fn drop(&mut self) {
+ self.raw.drop_abort_handle();
+ }
+}
diff --git a/src/runtime/task/core.rs b/src/runtime/task/core.rs
index 776e834..bcccc69 100644
--- a/src/runtime/task/core.rs
+++ b/src/runtime/task/core.rs
@@ -11,9 +11,10 @@
use crate::future::Future;
use crate::loom::cell::UnsafeCell;
+use crate::runtime::context;
use crate::runtime::task::raw::{self, Vtable};
use crate::runtime::task::state::State;
-use crate::runtime::task::Schedule;
+use crate::runtime::task::{Id, Schedule};
use crate::util::linked_list;
use std::pin::Pin;
@@ -24,6 +25,9 @@
///
/// It is critical for `Header` to be the first field as the task structure will
/// be referenced by both *mut Cell and *mut Header.
+///
+/// Any changes to the layout of this struct _must_ also be reflected in the
+/// const fns in raw.rs.
#[repr(C)]
pub(super) struct Cell<T: Future, S> {
/// Hot task state data
@@ -43,10 +47,17 @@
/// The core of the task.
///
/// Holds the future or output, depending on the stage of execution.
+///
+/// Any changes to the layout of this struct _must_ also be reflected in the
+/// const fns in raw.rs.
+#[repr(C)]
pub(super) struct Core<T: Future, S> {
/// Scheduler used to drive this future.
pub(super) scheduler: S,
+ /// The task's ID, used for populating `JoinError`s.
+ pub(super) task_id: Id,
+
/// Either the future or the output.
pub(super) stage: CoreStage<T>,
}
@@ -57,8 +68,6 @@
/// Task state.
pub(super) state: State,
- pub(super) owned: UnsafeCell<linked_list::Pointers<Header>>,
-
/// Pointer to next task, used with the injection queue.
pub(super) queue_next: UnsafeCell<Option<NonNull<Header>>>,
@@ -80,18 +89,29 @@
/// The tracing ID for this instrumented task.
#[cfg(all(tokio_unstable, feature = "tracing"))]
- pub(super) id: Option<tracing::Id>,
+ pub(super) tracing_id: Option<tracing::Id>,
}
unsafe impl Send for Header {}
unsafe impl Sync for Header {}
-/// Cold data is stored after the future.
+/// Cold data is stored after the future. Data is considered cold if it is only
+/// used during creation or shutdown of the task.
pub(super) struct Trailer {
+ /// Pointers for the linked list in the `OwnedTasks` that owns this task.
+ pub(super) owned: linked_list::Pointers<Header>,
/// Consumer task waiting on completion of this task.
pub(super) waker: UnsafeCell<Option<Waker>>,
}
+generate_addr_of_methods! {
+ impl<> Trailer {
+ pub(super) unsafe fn addr_of_owned(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Header>> {
+ &self.owned
+ }
+ }
+}
+
/// Either the future or the output.
pub(super) enum Stage<T: Future> {
Running(T),
@@ -102,29 +122,48 @@
impl<T: Future, S: Schedule> Cell<T, S> {
/// Allocates a new task cell, containing the header, trailer, and core
/// structures.
- pub(super) fn new(future: T, scheduler: S, state: State) -> Box<Cell<T, S>> {
+ pub(super) fn new(future: T, scheduler: S, state: State, task_id: Id) -> Box<Cell<T, S>> {
#[cfg(all(tokio_unstable, feature = "tracing"))]
- let id = future.id();
- Box::new(Cell {
+ let tracing_id = future.id();
+ let result = Box::new(Cell {
header: Header {
state,
- owned: UnsafeCell::new(linked_list::Pointers::new()),
queue_next: UnsafeCell::new(None),
vtable: raw::vtable::<T, S>(),
owner_id: UnsafeCell::new(0),
#[cfg(all(tokio_unstable, feature = "tracing"))]
- id,
+ tracing_id,
},
core: Core {
scheduler,
stage: CoreStage {
stage: UnsafeCell::new(Stage::Running(future)),
},
+ task_id,
},
trailer: Trailer {
waker: UnsafeCell::new(None),
+ owned: linked_list::Pointers::new(),
},
- })
+ });
+
+ #[cfg(debug_assertions)]
+ {
+ let trailer_addr = (&result.trailer) as *const Trailer as usize;
+ let trailer_ptr = unsafe { Header::get_trailer(NonNull::from(&result.header)) };
+ assert_eq!(trailer_addr, trailer_ptr.as_ptr() as usize);
+
+ let scheduler_addr = (&result.core.scheduler) as *const S as usize;
+ let scheduler_ptr =
+ unsafe { Header::get_scheduler::<S>(NonNull::from(&result.header)) };
+ assert_eq!(scheduler_addr, scheduler_ptr.as_ptr() as usize);
+
+ let id_addr = (&result.core.task_id) as *const Id as usize;
+ let id_ptr = unsafe { Header::get_id_ptr(NonNull::from(&result.header)) };
+ assert_eq!(id_addr, id_ptr.as_ptr() as usize);
+ }
+
+ result
}
}
@@ -132,7 +171,29 @@
pub(super) fn with_mut<R>(&self, f: impl FnOnce(*mut Stage<T>) -> R) -> R {
self.stage.with_mut(f)
}
+}
+/// Set and clear the task id in the context when the future is executed or
+/// dropped, or when the output produced by the future is dropped.
+pub(crate) struct TaskIdGuard {
+ parent_task_id: Option<Id>,
+}
+
+impl TaskIdGuard {
+ fn enter(id: Id) -> Self {
+ TaskIdGuard {
+ parent_task_id: context::set_current_task_id(Some(id)),
+ }
+ }
+}
+
+impl Drop for TaskIdGuard {
+ fn drop(&mut self) {
+ context::set_current_task_id(self.parent_task_id);
+ }
+}
+
+impl<T: Future, S: Schedule> Core<T, S> {
/// Polls the future.
///
/// # Safety
@@ -148,7 +209,7 @@
/// heap.
pub(super) fn poll(&self, mut cx: Context<'_>) -> Poll<T::Output> {
let res = {
- self.stage.with_mut(|ptr| {
+ self.stage.stage.with_mut(|ptr| {
// Safety: The caller ensures mutual exclusion to the field.
let future = match unsafe { &mut *ptr } {
Stage::Running(future) => future,
@@ -158,6 +219,7 @@
// Safety: The caller ensures the future is pinned.
let future = unsafe { Pin::new_unchecked(future) };
+ let _guard = TaskIdGuard::enter(self.task_id);
future.poll(&mut cx)
})
};
@@ -201,7 +263,7 @@
pub(super) fn take_output(&self) -> super::Result<T::Output> {
use std::mem;
- self.stage.with_mut(|ptr| {
+ self.stage.stage.with_mut(|ptr| {
// Safety:: the caller ensures mutual exclusion to the field.
match mem::replace(unsafe { &mut *ptr }, Stage::Consumed) {
Stage::Finished(output) => output,
@@ -211,7 +273,8 @@
}
unsafe fn set_stage(&self, stage: Stage<T>) {
- self.stage.with_mut(|ptr| *ptr = stage)
+ let _guard = TaskIdGuard::enter(self.task_id);
+ self.stage.stage.with_mut(|ptr| *ptr = stage)
}
}
@@ -236,6 +299,62 @@
// the safety requirements on `set_owner_id`.
unsafe { self.owner_id.with(|ptr| *ptr) }
}
+
+ /// Gets a pointer to the `Trailer` of the task containing this `Header`.
+ ///
+ /// # Safety
+ ///
+ /// The provided raw pointer must point at the header of a task.
+ pub(super) unsafe fn get_trailer(me: NonNull<Header>) -> NonNull<Trailer> {
+ let offset = me.as_ref().vtable.trailer_offset;
+ let trailer = me.as_ptr().cast::<u8>().add(offset).cast::<Trailer>();
+ NonNull::new_unchecked(trailer)
+ }
+
+ /// Gets a pointer to the scheduler of the task containing this `Header`.
+ ///
+ /// # Safety
+ ///
+ /// The provided raw pointer must point at the header of a task.
+ ///
+ /// The generic type S must be set to the correct scheduler type for this
+ /// task.
+ pub(super) unsafe fn get_scheduler<S>(me: NonNull<Header>) -> NonNull<S> {
+ let offset = me.as_ref().vtable.scheduler_offset;
+ let scheduler = me.as_ptr().cast::<u8>().add(offset).cast::<S>();
+ NonNull::new_unchecked(scheduler)
+ }
+
+ /// Gets a pointer to the id of the task containing this `Header`.
+ ///
+ /// # Safety
+ ///
+ /// The provided raw pointer must point at the header of a task.
+ pub(super) unsafe fn get_id_ptr(me: NonNull<Header>) -> NonNull<Id> {
+ let offset = me.as_ref().vtable.id_offset;
+ let id = me.as_ptr().cast::<u8>().add(offset).cast::<Id>();
+ NonNull::new_unchecked(id)
+ }
+
+ /// Gets the id of the task containing this `Header`.
+ ///
+ /// # Safety
+ ///
+ /// The provided raw pointer must point at the header of a task.
+ pub(super) unsafe fn get_id(me: NonNull<Header>) -> Id {
+ let ptr = Header::get_id_ptr(me).as_ptr();
+ *ptr
+ }
+
+ /// Gets the tracing id of the task containing this `Header`.
+ ///
+ /// # Safety
+ ///
+ /// The provided raw pointer must point at the header of a task.
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) unsafe fn get_tracing_id(me: &NonNull<Header>) -> Option<&tracing::Id> {
+ me.as_ref().tracing_id.as_ref()
+ }
}
impl Trailer {
diff --git a/src/runtime/task/error.rs b/src/runtime/task/error.rs
index 1a8129b..f7ead77 100644
--- a/src/runtime/task/error.rs
+++ b/src/runtime/task/error.rs
@@ -2,12 +2,13 @@
use std::fmt;
use std::io;
+use super::Id;
use crate::util::SyncWrapper;
-
cfg_rt! {
/// Task failed to execute to completion.
pub struct JoinError {
repr: Repr,
+ id: Id,
}
}
@@ -17,15 +18,17 @@
}
impl JoinError {
- pub(crate) fn cancelled() -> JoinError {
+ pub(crate) fn cancelled(id: Id) -> JoinError {
JoinError {
repr: Repr::Cancelled,
+ id,
}
}
- pub(crate) fn panic(err: Box<dyn Any + Send + 'static>) -> JoinError {
+ pub(crate) fn panic(id: Id, err: Box<dyn Any + Send + 'static>) -> JoinError {
JoinError {
repr: Repr::Panic(SyncWrapper::new(err)),
+ id,
}
}
@@ -79,6 +82,7 @@
/// }
/// }
/// ```
+ #[track_caller]
pub fn into_panic(self) -> Box<dyn Any + Send + 'static> {
self.try_into_panic()
.expect("`JoinError` reason is not a panic.")
@@ -111,13 +115,28 @@
_ => Err(self),
}
}
+
+ /// Returns a [task ID] that identifies the task which errored relative to
+ /// other currently spawned tasks.
+ ///
+ /// **Note**: This is an [unstable API][unstable]. The public API of this type
+ /// may break in 1.x releases. See [the documentation on unstable
+ /// features][unstable] for details.
+ ///
+ /// [task ID]: crate::task::Id
+ /// [unstable]: crate#unstable-features
+ #[cfg(tokio_unstable)]
+ #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
+ pub fn id(&self) -> Id {
+ self.id
+ }
}
impl fmt::Display for JoinError {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.repr {
- Repr::Cancelled => write!(fmt, "cancelled"),
- Repr::Panic(_) => write!(fmt, "panic"),
+ Repr::Cancelled => write!(fmt, "task {} was cancelled", self.id),
+ Repr::Panic(_) => write!(fmt, "task {} panicked", self.id),
}
}
}
@@ -125,8 +144,8 @@
impl fmt::Debug for JoinError {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.repr {
- Repr::Cancelled => write!(fmt, "JoinError::Cancelled"),
- Repr::Panic(_) => write!(fmt, "JoinError::Panic(...)"),
+ Repr::Cancelled => write!(fmt, "JoinError::Cancelled({:?})", self.id),
+ Repr::Panic(_) => write!(fmt, "JoinError::Panic({:?}, ...)", self.id),
}
}
}
diff --git a/src/runtime/task/harness.rs b/src/runtime/task/harness.rs
index 0996e52..8e3c3d1 100644
--- a/src/runtime/task/harness.rs
+++ b/src/runtime/task/harness.rs
@@ -1,8 +1,8 @@
use crate::future::Future;
-use crate::runtime::task::core::{Cell, Core, CoreStage, Header, Trailer};
-use crate::runtime::task::state::Snapshot;
+use crate::runtime::task::core::{Cell, Core, Header, Trailer};
+use crate::runtime::task::state::{Snapshot, State};
use crate::runtime::task::waker::waker_ref;
-use crate::runtime::task::{JoinError, Notified, Schedule, Task};
+use crate::runtime::task::{JoinError, Notified, RawTask, Schedule, Task};
use std::mem;
use std::mem::ManuallyDrop;
@@ -26,8 +26,16 @@
}
}
+ fn header_ptr(&self) -> NonNull<Header> {
+ self.cell.cast()
+ }
+
fn header(&self) -> &Header {
- unsafe { &self.cell.as_ref().header }
+ unsafe { &*self.header_ptr().as_ptr() }
+ }
+
+ fn state(&self) -> &State {
+ &self.header().state
}
fn trailer(&self) -> &Trailer {
@@ -39,11 +47,102 @@
}
}
+/// Task operations that can be implemented without being generic over the
+/// scheduler or task. Only one version of these methods should exist in the
+/// final binary.
+impl RawTask {
+ pub(super) fn drop_reference(self) {
+ if self.state().ref_dec() {
+ self.dealloc();
+ }
+ }
+
+ /// This call consumes a ref-count and notifies the task. This will create a
+ /// new Notified and submit it if necessary.
+ ///
+ /// The caller does not need to hold a ref-count besides the one that was
+ /// passed to this call.
+ pub(super) fn wake_by_val(&self) {
+ use super::state::TransitionToNotifiedByVal;
+
+ match self.state().transition_to_notified_by_val() {
+ TransitionToNotifiedByVal::Submit => {
+ // The caller has given us a ref-count, and the transition has
+ // created a new ref-count, so we now hold two. We turn the new
+ // ref-count Notified and pass it to the call to `schedule`.
+ //
+ // The old ref-count is retained for now to ensure that the task
+ // is not dropped during the call to `schedule` if the call
+ // drops the task it was given.
+ self.schedule();
+
+ // Now that we have completed the call to schedule, we can
+ // release our ref-count.
+ self.drop_reference();
+ }
+ TransitionToNotifiedByVal::Dealloc => {
+ self.dealloc();
+ }
+ TransitionToNotifiedByVal::DoNothing => {}
+ }
+ }
+
+ /// This call notifies the task. It will not consume any ref-counts, but the
+ /// caller should hold a ref-count. This will create a new Notified and
+ /// submit it if necessary.
+ pub(super) fn wake_by_ref(&self) {
+ use super::state::TransitionToNotifiedByRef;
+
+ match self.state().transition_to_notified_by_ref() {
+ TransitionToNotifiedByRef::Submit => {
+ // The transition above incremented the ref-count for a new task
+ // and the caller also holds a ref-count. The caller's ref-count
+ // ensures that the task is not destroyed even if the new task
+ // is dropped before `schedule` returns.
+ self.schedule();
+ }
+ TransitionToNotifiedByRef::DoNothing => {}
+ }
+ }
+
+ /// Remotely aborts the task.
+ ///
+ /// The caller should hold a ref-count, but we do not consume it.
+ ///
+ /// This is similar to `shutdown` except that it asks the runtime to perform
+ /// the shutdown. This is necessary to avoid the shutdown happening in the
+ /// wrong thread for non-Send tasks.
+ pub(super) fn remote_abort(&self) {
+ if self.state().transition_to_notified_and_cancel() {
+ // The transition has created a new ref-count, which we turn into
+ // a Notified and pass to the task.
+ //
+ // Since the caller holds a ref-count, the task cannot be destroyed
+ // before the call to `schedule` returns even if the call drops the
+ // `Notified` internally.
+ self.schedule();
+ }
+ }
+
+ /// Try to set the waker notified when the task is complete. Returns true if
+ /// the task has already completed. If this call returns false, then the
+ /// waker will not be notified.
+ pub(super) fn try_set_join_waker(&self, waker: &Waker) -> bool {
+ can_read_output(self.header(), self.trailer(), waker)
+ }
+}
+
impl<T, S> Harness<T, S>
where
T: Future,
S: Schedule,
{
+ pub(super) fn drop_reference(self) {
+ if self.state().ref_dec() {
+ self.dealloc();
+ }
+ }
+
/// Polls the inner future. A ref-count is consumed.
///
/// All necessary state checks and transitions are performed.
@@ -91,32 +190,32 @@
fn poll_inner(&self) -> PollFuture {
use super::state::{TransitionToIdle, TransitionToRunning};
- match self.header().state.transition_to_running() {
+ match self.state().transition_to_running() {
TransitionToRunning::Success => {
- let waker_ref = waker_ref::<T, S>(self.header());
- let cx = Context::from_waker(&*waker_ref);
- let res = poll_future(&self.core().stage, cx);
+ let header_ptr = self.header_ptr();
+ let waker_ref = waker_ref::<T, S>(&header_ptr);
+ let cx = Context::from_waker(&waker_ref);
+ let res = poll_future(self.core(), cx);
if res == Poll::Ready(()) {
// The future completed. Move on to complete the task.
return PollFuture::Complete;
}
- match self.header().state.transition_to_idle() {
+ match self.state().transition_to_idle() {
TransitionToIdle::Ok => PollFuture::Done,
TransitionToIdle::OkNotified => PollFuture::Notified,
TransitionToIdle::OkDealloc => PollFuture::Dealloc,
TransitionToIdle::Cancelled => {
// The transition to idle failed because the task was
// cancelled during the poll.
-
- cancel_task(&self.core().stage);
+ cancel_task(self.core());
PollFuture::Complete
}
}
}
TransitionToRunning::Cancelled => {
- cancel_task(&self.core().stage);
+ cancel_task(self.core());
PollFuture::Complete
}
TransitionToRunning::Failed => PollFuture::Done,
@@ -131,7 +230,7 @@
/// there is nothing further to do. When the task completes running, it will
/// notice the `CANCELLED` bit and finalize the task.
pub(super) fn shutdown(self) {
- if !self.header().state.transition_to_shutdown() {
+ if !self.state().transition_to_shutdown() {
// The task is concurrently running. No further work needed.
self.drop_reference();
return;
@@ -139,7 +238,7 @@
// By transitioning the lifecycle to `Running`, we have permission to
// drop the future.
- cancel_task(&self.core().stage);
+ cancel_task(self.core());
self.complete();
}
@@ -150,6 +249,19 @@
// Check causality
self.core().stage.with_mut(drop);
+ // Safety: The caller of this method just transitioned our ref-count to
+ // zero, so it is our responsibility to release the allocation.
+ //
+ // We don't hold any references into the allocation at this point, but
+ // it is possible for another thread to still hold a `&State` into the
+ // allocation if that other thread has decremented its last ref-count,
+ // but has not yet returned from the relevant method on `State`.
+ //
+ // However, the `State` type consists of just an `AtomicUsize`, and an
+ // `AtomicUsize` wraps the entirety of its contents in an `UnsafeCell`.
+ // As explained in the documentation for `UnsafeCell`, such references
+ // are allowed to be dangling after their last use, even if the
+ // reference has not yet gone out of scope.
unsafe {
drop(Box::from_raw(self.cell.as_ptr()));
}
@@ -160,122 +272,30 @@
/// Read the task output into `dst`.
pub(super) fn try_read_output(self, dst: &mut Poll<super::Result<T::Output>>, waker: &Waker) {
if can_read_output(self.header(), self.trailer(), waker) {
- *dst = Poll::Ready(self.core().stage.take_output());
+ *dst = Poll::Ready(self.core().take_output());
}
}
pub(super) fn drop_join_handle_slow(self) {
- let mut maybe_panic = None;
-
// Try to unset `JOIN_INTEREST`. This must be done as a first step in
// case the task concurrently completed.
- if self.header().state.unset_join_interested().is_err() {
+ if self.state().unset_join_interested().is_err() {
// It is our responsibility to drop the output. This is critical as
// the task output may not be `Send` and as such must remain with
// the scheduler or `JoinHandle`. i.e. if the output remains in the
// task structure until the task is deallocated, it may be dropped
// by a Waker on any arbitrary thread.
- let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
- self.core().stage.drop_future_or_output();
+ //
+ // Panics are delivered to the user via the `JoinHandle`. Given that
+ // they are dropping the `JoinHandle`, we assume they are not
+ // interested in the panic and swallow it.
+ let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+ self.core().drop_future_or_output();
}));
-
- if let Err(panic) = panic {
- maybe_panic = Some(panic);
- }
}
// Drop the `JoinHandle` reference, possibly deallocating the task
self.drop_reference();
-
- if let Some(panic) = maybe_panic {
- panic::resume_unwind(panic);
- }
- }
-
- /// Remotely aborts the task.
- ///
- /// The caller should hold a ref-count, but we do not consume it.
- ///
- /// This is similar to `shutdown` except that it asks the runtime to perform
- /// the shutdown. This is necessary to avoid the shutdown happening in the
- /// wrong thread for non-Send tasks.
- pub(super) fn remote_abort(self) {
- if self.header().state.transition_to_notified_and_cancel() {
- // The transition has created a new ref-count, which we turn into
- // a Notified and pass to the task.
- //
- // Since the caller holds a ref-count, the task cannot be destroyed
- // before the call to `schedule` returns even if the call drops the
- // `Notified` internally.
- self.core()
- .scheduler
- .schedule(Notified(self.get_new_task()));
- }
- }
-
- // ===== waker behavior =====
-
- /// This call consumes a ref-count and notifies the task. This will create a
- /// new Notified and submit it if necessary.
- ///
- /// The caller does not need to hold a ref-count besides the one that was
- /// passed to this call.
- pub(super) fn wake_by_val(self) {
- use super::state::TransitionToNotifiedByVal;
-
- match self.header().state.transition_to_notified_by_val() {
- TransitionToNotifiedByVal::Submit => {
- // The caller has given us a ref-count, and the transition has
- // created a new ref-count, so we now hold two. We turn the new
- // ref-count Notified and pass it to the call to `schedule`.
- //
- // The old ref-count is retained for now to ensure that the task
- // is not dropped during the call to `schedule` if the call
- // drops the task it was given.
- self.core()
- .scheduler
- .schedule(Notified(self.get_new_task()));
-
- // Now that we have completed the call to schedule, we can
- // release our ref-count.
- self.drop_reference();
- }
- TransitionToNotifiedByVal::Dealloc => {
- self.dealloc();
- }
- TransitionToNotifiedByVal::DoNothing => {}
- }
- }
-
- /// This call notifies the task. It will not consume any ref-counts, but the
- /// caller should hold a ref-count. This will create a new Notified and
- /// submit it if necessary.
- pub(super) fn wake_by_ref(&self) {
- use super::state::TransitionToNotifiedByRef;
-
- match self.header().state.transition_to_notified_by_ref() {
- TransitionToNotifiedByRef::Submit => {
- // The transition above incremented the ref-count for a new task
- // and the caller also holds a ref-count. The caller's ref-count
- // ensures that the task is not destroyed even if the new task
- // is dropped before `schedule` returns.
- self.core()
- .scheduler
- .schedule(Notified(self.get_new_task()));
- }
- TransitionToNotifiedByRef::DoNothing => {}
- }
- }
-
- pub(super) fn drop_reference(self) {
- if self.header().state.ref_dec() {
- self.dealloc();
- }
- }
-
- #[cfg(all(tokio_unstable, feature = "tracing"))]
- pub(super) fn id(&self) -> Option<&tracing::Id> {
- self.header().id.as_ref()
}
// ====== internal ======
@@ -285,7 +305,7 @@
// The future has completed and its output has been written to the task
// stage. We transition from running to complete.
- let snapshot = self.header().state.transition_to_complete();
+ let snapshot = self.state().transition_to_complete();
// We catch panics here in case dropping the future or waking the
// JoinHandle panics.
@@ -294,10 +314,11 @@
// The `JoinHandle` is not interested in the output of
// this task. It is our responsibility to drop the
// output.
- self.core().stage.drop_future_or_output();
- } else if snapshot.has_join_waker() {
- // Notify the join handle. The previous transition obtains the
- // lock on the waker cell.
+ self.core().drop_future_or_output();
+ } else if snapshot.is_join_waker_set() {
+ // Notify the waker. Reading the waker field is safe per rule 4
+ // in task/mod.rs, since the JOIN_WAKER bit is set and the call
+ // to transition_to_complete() above set the COMPLETE bit.
self.trailer().wake_join();
}
}));
@@ -305,7 +326,7 @@
// The task has completed execution and will no longer be scheduled.
let num_release = self.release();
- if self.header().state.transition_to_terminal(num_release) {
+ if self.state().transition_to_terminal(num_release) {
self.dealloc();
}
}
@@ -347,36 +368,30 @@
debug_assert!(snapshot.is_join_interested());
if !snapshot.is_complete() {
- // The waker must be stored in the task struct.
- let res = if snapshot.has_join_waker() {
- // There already is a waker stored in the struct. If it matches
- // the provided waker, then there is no further work to do.
- // Otherwise, the waker must be swapped.
- let will_wake = unsafe {
- // Safety: when `JOIN_INTEREST` is set, only `JOIN_HANDLE`
- // may mutate the `waker` field.
- trailer.will_wake(waker)
- };
+ // If the task is not complete, try storing the provided waker in the
+ // task's waker field.
- if will_wake {
- // The task is not complete **and** the waker is up to date,
- // there is nothing further that needs to be done.
+ let res = if snapshot.is_join_waker_set() {
+ // If JOIN_WAKER is set, then JoinHandle has previously stored a
+ // waker in the waker field per step (iii) of rule 5 in task/mod.rs.
+
+ // Optimization: if the stored waker and the provided waker wake the
+ // same task, then return without touching the waker field. (Reading
+ // the waker field below is safe per rule 3 in task/mod.rs.)
+ if unsafe { trailer.will_wake(waker) } {
return false;
}
- // Unset the `JOIN_WAKER` to gain mutable access to the `waker`
- // field then update the field with the new join worker.
- //
- // This requires two atomic operations, unsetting the bit and
- // then resetting it. If the task transitions to complete
- // concurrently to either one of those operations, then setting
- // the join waker fails and we proceed to reading the task
- // output.
+ // Otherwise swap the stored waker with the provided waker by
+ // following the rule 5 in task/mod.rs.
header
.state
.unset_waker()
.and_then(|snapshot| set_join_waker(header, trailer, waker.clone(), snapshot))
} else {
+ // If JOIN_WAKER is unset, then JoinHandle has mutable access to the
+ // waker field per rule 2 in task/mod.rs; therefore, skip step (i)
+ // of rule 5 and try to store the provided waker in the waker field.
set_join_waker(header, trailer, waker.clone(), snapshot)
};
@@ -397,7 +412,7 @@
snapshot: Snapshot,
) -> Result<Snapshot, Snapshot> {
assert!(snapshot.is_join_interested());
- assert!(!snapshot.has_join_waker());
+ assert!(!snapshot.is_join_waker_set());
// Safety: Only the `JoinHandle` may set the `waker` field. When
// `JOIN_INTEREST` is **not** set, nothing else will touch the field.
@@ -426,31 +441,31 @@
}
/// Cancels the task and store the appropriate error in the stage field.
-fn cancel_task<T: Future>(stage: &CoreStage<T>) {
+fn cancel_task<T: Future, S: Schedule>(core: &Core<T, S>) {
// Drop the future from a panic guard.
let res = panic::catch_unwind(panic::AssertUnwindSafe(|| {
- stage.drop_future_or_output();
+ core.drop_future_or_output();
}));
match res {
Ok(()) => {
- stage.store_output(Err(JoinError::cancelled()));
+ core.store_output(Err(JoinError::cancelled(core.task_id)));
}
Err(panic) => {
- stage.store_output(Err(JoinError::panic(panic)));
+ core.store_output(Err(JoinError::panic(core.task_id, panic)));
}
}
}
/// Polls the future. If the future completes, the output is written to the
/// stage field.
-fn poll_future<T: Future>(core: &CoreStage<T>, cx: Context<'_>) -> Poll<()> {
+fn poll_future<T: Future, S: Schedule>(core: &Core<T, S>, cx: Context<'_>) -> Poll<()> {
// Poll the future.
let output = panic::catch_unwind(panic::AssertUnwindSafe(|| {
- struct Guard<'a, T: Future> {
- core: &'a CoreStage<T>,
+ struct Guard<'a, T: Future, S: Schedule> {
+ core: &'a Core<T, S>,
}
- impl<'a, T: Future> Drop for Guard<'a, T> {
+ impl<'a, T: Future, S: Schedule> Drop for Guard<'a, T, S> {
fn drop(&mut self) {
// If the future panics on poll, we drop it inside the panic
// guard.
@@ -467,13 +482,20 @@
let output = match output {
Ok(Poll::Pending) => return Poll::Pending,
Ok(Poll::Ready(output)) => Ok(output),
- Err(panic) => Err(JoinError::panic(panic)),
+ Err(panic) => {
+ core.scheduler.unhandled_panic();
+ Err(JoinError::panic(core.task_id, panic))
+ }
};
// Catch and ignore panics if the future panics on drop.
- let _ = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+ let res = panic::catch_unwind(panic::AssertUnwindSafe(|| {
core.store_output(output);
}));
+ if res.is_err() {
+ core.scheduler.unhandled_panic();
+ }
+
Poll::Ready(())
}
diff --git a/src/runtime/task/id.rs b/src/runtime/task/id.rs
new file mode 100644
index 0000000..2b0d95c
--- /dev/null
+++ b/src/runtime/task/id.rs
@@ -0,0 +1,87 @@
+use crate::runtime::context;
+
+use std::fmt;
+
+/// An opaque ID that uniquely identifies a task relative to all other currently
+/// running tasks.
+///
+/// # Notes
+///
+/// - Task IDs are unique relative to other *currently running* tasks. When a
+/// task completes, the same ID may be used for another task.
+/// - Task IDs are *not* sequential, and do not indicate the order in which
+/// tasks are spawned, what runtime a task is spawned on, or any other data.
+/// - The task ID of the currently running task can be obtained from inside the
+/// task via the [`task::try_id()`](crate::task::try_id()) and
+/// [`task::id()`](crate::task::id()) functions and from outside the task via
+/// the [`JoinHandle::id()`](crate::task::JoinHandle::id()) function.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [unstable]: crate#unstable-features
+#[cfg_attr(docsrs, doc(cfg(all(feature = "rt", tokio_unstable))))]
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
+pub struct Id(u64);
+
+/// Returns the [`Id`] of the currently running task.
+///
+/// # Panics
+///
+/// This function panics if called from outside a task. Please note that calls
+/// to `block_on` do not have task IDs, so the method will panic if called from
+/// within a call to `block_on`. For a version of this function that doesn't
+/// panic, see [`task::try_id()`](crate::runtime::task::try_id()).
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [task ID]: crate::task::Id
+/// [unstable]: crate#unstable-features
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[track_caller]
+pub fn id() -> Id {
+ context::current_task_id().expect("Can't get a task id when not inside a task")
+}
+
+/// Returns the [`Id`] of the currently running task, or `None` if called outside
+/// of a task.
+///
+/// This function is similar to [`task::id()`](crate::runtime::task::id()), except
+/// that it returns `None` rather than panicking if called outside of a task
+/// context.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [task ID]: crate::task::Id
+/// [unstable]: crate#unstable-features
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+#[track_caller]
+pub fn try_id() -> Option<Id> {
+ context::current_task_id()
+}
+
+impl fmt::Display for Id {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl Id {
+ pub(crate) fn next() -> Self {
+ use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64};
+
+ static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(1);
+
+ Self(NEXT_ID.fetch_add(1, Relaxed))
+ }
+
+ pub(crate) fn as_u64(&self) -> u64 {
+ self.0
+ }
+}
diff --git a/src/runtime/task/join.rs b/src/runtime/task/join.rs
index 0abbff2..11c4b9b 100644
--- a/src/runtime/task/join.rs
+++ b/src/runtime/task/join.rs
@@ -1,16 +1,19 @@
-use crate::runtime::task::RawTask;
+use crate::runtime::task::{Header, RawTask};
use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
+use std::panic::{RefUnwindSafe, UnwindSafe};
use std::pin::Pin;
-use std::task::{Context, Poll};
+use std::task::{Context, Poll, Waker};
cfg_rt! {
/// An owned permission to join on a task (await its termination).
///
- /// This can be thought of as the equivalent of [`std::thread::JoinHandle`] for
- /// a task rather than a thread.
+ /// This can be thought of as the equivalent of [`std::thread::JoinHandle`]
+ /// for a Tokio task rather than a thread. Note that the background task
+ /// associated with this `JoinHandle` started running immediately when you
+ /// called spawn, even if you have not yet awaited the `JoinHandle`.
///
/// A `JoinHandle` *detaches* the associated task when it is dropped, which
/// means that there is no longer any handle to the task, and no way to `join`
@@ -19,6 +22,15 @@
/// This `struct` is created by the [`task::spawn`] and [`task::spawn_blocking`]
/// functions.
///
+ /// # Cancel safety
+ ///
+ /// The `&mut JoinHandle<T>` type is cancel safe. If it is used as the event
+ /// in a `tokio::select!` statement and some other branch completes first,
+ /// then it is guaranteed that the output of the task is not lost.
+ ///
+ /// If a `JoinHandle` is dropped, then the task continues running in the
+ /// background and its return value is lost.
+ ///
/// # Examples
///
/// Creation from [`task::spawn`]:
@@ -142,7 +154,7 @@
/// [`std::thread::JoinHandle`]: std::thread::JoinHandle
/// [`JoinError`]: crate::task::JoinError
pub struct JoinHandle<T> {
- raw: Option<RawTask>,
+ raw: RawTask,
_p: PhantomData<T>,
}
}
@@ -150,10 +162,13 @@
unsafe impl<T: Send> Send for JoinHandle<T> {}
unsafe impl<T: Send> Sync for JoinHandle<T> {}
+impl<T> UnwindSafe for JoinHandle<T> {}
+impl<T> RefUnwindSafe for JoinHandle<T> {}
+
impl<T> JoinHandle<T> {
pub(super) fn new(raw: RawTask) -> JoinHandle<T> {
JoinHandle {
- raw: Some(raw),
+ raw,
_p: PhantomData,
}
}
@@ -192,10 +207,71 @@
/// ```
/// [cancelled]: method@super::error::JoinError::is_cancelled
pub fn abort(&self) {
- if let Some(raw) = self.raw {
- raw.remote_abort();
+ self.raw.remote_abort();
+ }
+
+ /// Checks if the task associated with this `JoinHandle` has finished.
+ ///
+ /// Please note that this method can return `false` even if [`abort`] has been
+ /// called on the task. This is because the cancellation process may take
+ /// some time, and this method does not return `true` until it has
+ /// completed.
+ ///
+ /// ```rust
+ /// use tokio::time;
+ ///
+ /// # #[tokio::main(flavor = "current_thread")]
+ /// # async fn main() {
+ /// # time::pause();
+ /// let handle1 = tokio::spawn(async {
+ /// // do some stuff here
+ /// });
+ /// let handle2 = tokio::spawn(async {
+ /// // do some other stuff here
+ /// time::sleep(time::Duration::from_secs(10)).await;
+ /// });
+ /// // Wait for the task to finish
+ /// handle2.abort();
+ /// time::sleep(time::Duration::from_secs(1)).await;
+ /// assert!(handle1.is_finished());
+ /// assert!(handle2.is_finished());
+ /// # }
+ /// ```
+ /// [`abort`]: method@JoinHandle::abort
+ pub fn is_finished(&self) -> bool {
+ let state = self.raw.header().state.load();
+ state.is_complete()
+ }
+
+ /// Set the waker that is notified when the task completes.
+ pub(crate) fn set_join_waker(&mut self, waker: &Waker) {
+ if self.raw.try_set_join_waker(waker) {
+ // In this case the task has already completed. We wake the waker immediately.
+ waker.wake_by_ref();
}
}
+
+ /// Returns a new `AbortHandle` that can be used to remotely abort this task.
+ pub(crate) fn abort_handle(&self) -> super::AbortHandle {
+ self.raw.ref_inc();
+ super::AbortHandle::new(self.raw)
+ }
+
+ /// Returns a [task ID] that uniquely identifies this task relative to other
+ /// currently spawned tasks.
+ ///
+ /// **Note**: This is an [unstable API][unstable]. The public API of this type
+ /// may break in 1.x releases. See [the documentation on unstable
+ /// features][unstable] for details.
+ ///
+ /// [task ID]: crate::task::Id
+ /// [unstable]: crate#unstable-features
+ #[cfg(tokio_unstable)]
+ #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
+ pub fn id(&self) -> super::Id {
+ // Safety: The header pointer is valid.
+ unsafe { Header::get_id(self.raw.header_ptr()) }
+ }
}
impl<T> Unpin for JoinHandle<T> {}
@@ -207,14 +283,7 @@
let mut ret = Poll::Pending;
// Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
-
- // Raw should always be set. If it is not, this is due to polling after
- // completion
- let raw = self
- .raw
- .as_ref()
- .expect("polling after `JoinHandle` already completed");
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
// Try to read the task output. If the task is not yet complete, the
// waker is stored and is notified once the task does complete.
@@ -228,7 +297,8 @@
//
// The type of `T` must match the task's output type.
unsafe {
- raw.try_read_output(&mut ret as *mut _ as *mut (), cx.waker());
+ self.raw
+ .try_read_output(&mut ret as *mut _ as *mut (), cx.waker());
}
if ret.is_ready() {
@@ -241,13 +311,11 @@
impl<T> Drop for JoinHandle<T> {
fn drop(&mut self) {
- if let Some(raw) = self.raw.take() {
- if raw.header().state.drop_join_handle_fast().is_ok() {
- return;
- }
-
- raw.drop_join_handle_slow();
+ if self.raw.state().drop_join_handle_fast().is_ok() {
+ return;
}
+
+ self.raw.drop_join_handle_slow();
}
}
@@ -256,6 +324,9 @@
T: fmt::Debug,
{
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt.debug_struct("JoinHandle").finish()
+ // Safety: The header pointer is valid.
+ let id_ptr = unsafe { Header::get_id_ptr(self.raw.header_ptr()) };
+ let id = unsafe { id_ptr.as_ref() };
+ fmt.debug_struct("JoinHandle").field("id", id).finish()
}
}
diff --git a/src/runtime/task/list.rs b/src/runtime/task/list.rs
index 7758f8d..159c13e 100644
--- a/src/runtime/task/list.rs
+++ b/src/runtime/task/list.rs
@@ -84,13 +84,14 @@
&self,
task: T,
scheduler: S,
+ id: super::Id,
) -> (JoinHandle<T::Output>, Option<Notified<S>>)
where
S: Schedule,
T: Future + Send + 'static,
T::Output: Send + 'static,
{
- let (task, notified, join) = super::new_task(task, scheduler);
+ let (task, notified, join) = super::new_task(task, scheduler, id);
unsafe {
// safety: We just created the task, so we have exclusive access
@@ -163,7 +164,7 @@
// safety: We just checked that the provided task is not in some other
// linked list.
- unsafe { self.inner.lock().list.remove(task.header().into()) }
+ unsafe { self.inner.lock().list.remove(task.header_ptr()) }
}
pub(crate) fn is_empty(&self) -> bool {
@@ -187,13 +188,14 @@
&self,
task: T,
scheduler: S,
+ id: super::Id,
) -> (JoinHandle<T::Output>, Option<Notified<S>>)
where
S: Schedule,
T: Future + 'static,
T::Output: 'static,
{
- let (task, notified, join) = super::new_task(task, scheduler);
+ let (task, notified, join) = super::new_task(task, scheduler, id);
unsafe {
// safety: We just created the task, so we have exclusive access
@@ -238,7 +240,7 @@
self.with_inner(|inner|
// safety: We just checked that the provided task is not in some
// other linked list.
- unsafe { inner.list.remove(task.header().into()) })
+ unsafe { inner.list.remove(task.header_ptr()) })
}
/// Asserts that the given task is owned by this LocalOwnedTasks and convert
diff --git a/src/runtime/task/mod.rs b/src/runtime/task/mod.rs
index 1f18209..55131ac 100644
--- a/src/runtime/task/mod.rs
+++ b/src/runtime/task/mod.rs
@@ -25,7 +25,7 @@
//!
//! The task uses a reference count to keep track of how many active references
//! exist. The Unowned reference type takes up two ref-counts. All other
-//! reference types take pu a single ref-count.
+//! reference types take up a single ref-count.
//!
//! Besides the waker type, each task has at most one of each reference type.
//!
@@ -47,7 +47,8 @@
//!
//! * JOIN_INTEREST - Is set to one if there exists a JoinHandle.
//!
-//! * JOIN_WAKER - Is set to one if the JoinHandle has set a waker.
+//! * JOIN_WAKER - Acts as an access control bit for the join handle waker. The
+//! protocol for its usage is described below.
//!
//! The rest of the bits are used for the ref-count.
//!
@@ -71,10 +72,38 @@
//! a lock for the stage field, and it can be accessed only by the thread
//! that set RUNNING to one.
//!
-//! * If JOIN_WAKER is zero, then the JoinHandle has exclusive access to the
-//! join handle waker. If JOIN_WAKER and COMPLETE are both one, then the
-//! thread that set COMPLETE to one has exclusive access to the join handle
-//! waker.
+//! * The waker field may be concurrently accessed by different threads: in one
+//! thread the runtime may complete a task and *read* the waker field to
+//! invoke the waker, and in another thread the task's JoinHandle may be
+//! polled, and if the task hasn't yet completed, the JoinHandle may *write*
+//! a waker to the waker field. The JOIN_WAKER bit ensures safe access by
+//! multiple threads to the waker field using the following rules:
+//!
+//! 1. JOIN_WAKER is initialized to zero.
+//!
+//! 2. If JOIN_WAKER is zero, then the JoinHandle has exclusive (mutable)
+//! access to the waker field.
+//!
+//! 3. If JOIN_WAKER is one, then the JoinHandle has shared (read-only)
+//! access to the waker field.
+//!
+//! 4. If JOIN_WAKER is one and COMPLETE is one, then the runtime has shared
+//! (read-only) access to the waker field.
+//!
+//! 5. If the JoinHandle needs to write to the waker field, then the
+//! JoinHandle needs to (i) successfully set JOIN_WAKER to zero if it is
+//! not already zero to gain exclusive access to the waker field per rule
+//! 2, (ii) write a waker, and (iii) successfully set JOIN_WAKER to one.
+//!
+//! 6. The JoinHandle can change JOIN_WAKER only if COMPLETE is zero (i.e.
+//! the task hasn't yet completed).
+//!
+//! Rule 6 implies that the steps (i) or (iii) of rule 5 may fail due to a
+//! race. If step (i) fails, then the attempt to write a waker is aborted. If
+//! step (iii) fails because COMPLETE is set to one by another thread after
+//! step (i), then the waker field is cleared. Once COMPLETE is one (i.e.
+//! task has completed), the JoinHandle will not modify JOIN_WAKER. After the
+//! runtime sets COMPLETE to one, it invokes the waker if there is one.
//!
//! All other fields are immutable and can be accessed immutably without
//! synchronization by anyone.
@@ -121,7 +150,7 @@
//! 1. The output is created on the thread that the future was polled on. Since
//! only non-Send futures can have non-Send output, the future was polled on
//! the thread that the future was spawned from.
-//! 2. Since JoinHandle<Output> is not Send if Output is not Send, the
+//! 2. Since `JoinHandle<Output>` is not Send if Output is not Send, the
//! JoinHandle is also on the thread that the future was spawned from.
//! 3. Thus, the JoinHandle will not move the output across threads when it
//! takes or drops the output.
@@ -135,24 +164,36 @@
//! poll call will notice it when the poll finishes, and the task is cancelled
//! at that point.
+// Some task infrastructure is here to support `JoinSet`, which is currently
+// unstable. This should be removed once `JoinSet` is stabilized.
+#![cfg_attr(not(tokio_unstable), allow(dead_code))]
+
mod core;
use self::core::Cell;
use self::core::Header;
mod error;
-#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411
pub use self::error::JoinError;
mod harness;
use self::harness::Harness;
+mod id;
+#[cfg_attr(not(tokio_unstable), allow(unreachable_pub))]
+pub use id::{id, try_id, Id};
+
cfg_rt_multi_thread! {
mod inject;
pub(super) use self::inject::Inject;
}
+#[cfg(feature = "rt")]
+mod abort;
mod join;
-#[allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411
+
+#[cfg(feature = "rt")]
+pub use self::abort::AbortHandle;
+
pub use self::join::JoinHandle;
mod list;
@@ -230,6 +271,11 @@
fn yield_now(&self, task: Notified<Self>) {
self.schedule(task);
}
+
+ /// Polling the task resulted in a panic. Should the runtime shutdown?
+ fn unhandled_panic(&self) {
+ // By default, do nothing. This maintains the 1.0 behavior.
+ }
}
cfg_rt! {
@@ -239,14 +285,15 @@
/// notification.
fn new_task<T, S>(
task: T,
- scheduler: S
+ scheduler: S,
+ id: Id,
) -> (Task<S>, Notified<S>, JoinHandle<T::Output>)
where
S: Schedule,
T: Future + 'static,
T::Output: 'static,
{
- let raw = RawTask::new::<T, S>(task, scheduler);
+ let raw = RawTask::new::<T, S>(task, scheduler, id);
let task = Task {
raw,
_p: PhantomData,
@@ -264,13 +311,13 @@
/// only when the task is not going to be stored in an `OwnedTasks` list.
///
/// Currently only blocking tasks use this method.
- pub(crate) fn unowned<T, S>(task: T, scheduler: S) -> (UnownedTask<S>, JoinHandle<T::Output>)
+ pub(crate) fn unowned<T, S>(task: T, scheduler: S, id: Id) -> (UnownedTask<S>, JoinHandle<T::Output>)
where
S: Schedule,
T: Send + Future + 'static,
T::Output: Send + 'static,
{
- let (task, notified, join) = new_task(task, scheduler);
+ let (task, notified, join) = new_task(task, scheduler, id);
// This transfers the ref-count of task and notified into an UnownedTask.
// This is valid because an UnownedTask holds two ref-counts.
@@ -296,6 +343,10 @@
fn header(&self) -> &Header {
self.raw.header()
}
+
+ fn header_ptr(&self) -> NonNull<Header> {
+ self.raw.header_ptr()
+ }
}
impl<S: 'static> Notified<S> {
@@ -313,7 +364,7 @@
impl<S: 'static> Task<S> {
fn into_raw(self) -> NonNull<Header> {
- let ret = self.header().into();
+ let ret = self.raw.header_ptr();
mem::forget(self);
ret
}
@@ -327,7 +378,7 @@
}
impl<S: Schedule> Task<S> {
- /// Pre-emptively cancels the task as part of the shutdown process.
+ /// Preemptively cancels the task as part of the shutdown process.
pub(crate) fn shutdown(self) {
let raw = self.raw;
mem::forget(self);
@@ -347,6 +398,7 @@
impl<S: Schedule> UnownedTask<S> {
// Used in test of the inject queue.
#[cfg(test)]
+ #[cfg_attr(tokio_wasm, allow(dead_code))]
pub(super) fn into_notified(self) -> Notified<S> {
Notified(self.into_task())
}
@@ -426,7 +478,7 @@
type Target = Header;
fn as_raw(handle: &Task<S>) -> NonNull<Header> {
- handle.header().into()
+ handle.raw.header_ptr()
}
unsafe fn from_raw(ptr: NonNull<Header>) -> Task<S> {
@@ -434,7 +486,6 @@
}
unsafe fn pointers(target: NonNull<Header>) -> NonNull<linked_list::Pointers<Header>> {
- // Not super great as it avoids some of looms checking...
- NonNull::from(target.as_ref().owned.with_mut(|ptr| &mut *ptr))
+ self::core::Trailer::addr_of_owned(Header::get_trailer(target))
}
}
diff --git a/src/runtime/task/raw.rs b/src/runtime/task/raw.rs
index fbc9574..b9700ae 100644
--- a/src/runtime/task/raw.rs
+++ b/src/runtime/task/raw.rs
@@ -1,5 +1,6 @@
use crate::future::Future;
-use crate::runtime::task::{Cell, Harness, Header, Schedule, State};
+use crate::runtime::task::core::{Core, Trailer};
+use crate::runtime::task::{Cell, Harness, Header, Id, Schedule, State};
use std::ptr::NonNull;
use std::task::{Poll, Waker};
@@ -13,6 +14,9 @@
/// Polls the future.
pub(super) poll: unsafe fn(NonNull<Header>),
+ /// Schedules the task for execution on the runtime.
+ pub(super) schedule: unsafe fn(NonNull<Header>),
+
/// Deallocates the memory.
pub(super) dealloc: unsafe fn(NonNull<Header>),
@@ -22,32 +26,142 @@
/// The join handle has been dropped.
pub(super) drop_join_handle_slow: unsafe fn(NonNull<Header>),
- /// The task is remotely aborted.
- pub(super) remote_abort: unsafe fn(NonNull<Header>),
+ /// An abort handle has been dropped.
+ pub(super) drop_abort_handle: unsafe fn(NonNull<Header>),
/// Scheduler is being shutdown.
pub(super) shutdown: unsafe fn(NonNull<Header>),
+
+ /// The number of bytes that the `trailer` field is offset from the header.
+ pub(super) trailer_offset: usize,
+
+ /// The number of bytes that the `scheduler` field is offset from the header.
+ pub(super) scheduler_offset: usize,
+
+ /// The number of bytes that the `id` field is offset from the header.
+ pub(super) id_offset: usize,
}
/// Get the vtable for the requested `T` and `S` generics.
pub(super) fn vtable<T: Future, S: Schedule>() -> &'static Vtable {
&Vtable {
poll: poll::<T, S>,
+ schedule: schedule::<S>,
dealloc: dealloc::<T, S>,
try_read_output: try_read_output::<T, S>,
drop_join_handle_slow: drop_join_handle_slow::<T, S>,
- remote_abort: remote_abort::<T, S>,
+ drop_abort_handle: drop_abort_handle::<T, S>,
shutdown: shutdown::<T, S>,
+ trailer_offset: OffsetHelper::<T, S>::TRAILER_OFFSET,
+ scheduler_offset: OffsetHelper::<T, S>::SCHEDULER_OFFSET,
+ id_offset: OffsetHelper::<T, S>::ID_OFFSET,
}
}
+/// Calling `get_trailer_offset` directly in vtable doesn't work because it
+/// prevents the vtable from being promoted to a static reference.
+///
+/// See this thread for more info:
+/// <https://users.rust-lang.org/t/custom-vtables-with-integers/78508>
+struct OffsetHelper<T, S>(T, S);
+impl<T: Future, S: Schedule> OffsetHelper<T, S> {
+ // Pass `size_of`/`align_of` as arguments rather than calling them directly
+ // inside `get_trailer_offset` because trait bounds on generic parameters
+ // of const fn are unstable on our MSRV.
+ const TRAILER_OFFSET: usize = get_trailer_offset(
+ std::mem::size_of::<Header>(),
+ std::mem::size_of::<Core<T, S>>(),
+ std::mem::align_of::<Core<T, S>>(),
+ std::mem::align_of::<Trailer>(),
+ );
+
+ // The `scheduler` is the first field of `Core`, so it has the same
+ // offset as `Core`.
+ const SCHEDULER_OFFSET: usize = get_core_offset(
+ std::mem::size_of::<Header>(),
+ std::mem::align_of::<Core<T, S>>(),
+ );
+
+ const ID_OFFSET: usize = get_id_offset(
+ std::mem::size_of::<Header>(),
+ std::mem::align_of::<Core<T, S>>(),
+ std::mem::size_of::<S>(),
+ std::mem::align_of::<Id>(),
+ );
+}
+
+/// Compute the offset of the `Trailer` field in `Cell<T, S>` using the
+/// `#[repr(C)]` algorithm.
+///
+/// Pseudo-code for the `#[repr(C)]` algorithm can be found here:
+/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs>
+const fn get_trailer_offset(
+ header_size: usize,
+ core_size: usize,
+ core_align: usize,
+ trailer_align: usize,
+) -> usize {
+ let mut offset = header_size;
+
+ let core_misalign = offset % core_align;
+ if core_misalign > 0 {
+ offset += core_align - core_misalign;
+ }
+ offset += core_size;
+
+ let trailer_misalign = offset % trailer_align;
+ if trailer_misalign > 0 {
+ offset += trailer_align - trailer_misalign;
+ }
+
+ offset
+}
+
+/// Compute the offset of the `Core<T, S>` field in `Cell<T, S>` using the
+/// `#[repr(C)]` algorithm.
+///
+/// Pseudo-code for the `#[repr(C)]` algorithm can be found here:
+/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs>
+const fn get_core_offset(header_size: usize, core_align: usize) -> usize {
+ let mut offset = header_size;
+
+ let core_misalign = offset % core_align;
+ if core_misalign > 0 {
+ offset += core_align - core_misalign;
+ }
+
+ offset
+}
+
+/// Compute the offset of the `Id` field in `Cell<T, S>` using the
+/// `#[repr(C)]` algorithm.
+///
+/// Pseudo-code for the `#[repr(C)]` algorithm can be found here:
+/// <https://doc.rust-lang.org/reference/type-layout.html#reprc-structs>
+const fn get_id_offset(
+ header_size: usize,
+ core_align: usize,
+ scheduler_size: usize,
+ id_align: usize,
+) -> usize {
+ let mut offset = get_core_offset(header_size, core_align);
+ offset += scheduler_size;
+
+ let id_misalign = offset % id_align;
+ if id_misalign > 0 {
+ offset += id_align - id_misalign;
+ }
+
+ offset
+}
+
impl RawTask {
- pub(super) fn new<T, S>(task: T, scheduler: S) -> RawTask
+ pub(super) fn new<T, S>(task: T, scheduler: S, id: Id) -> RawTask
where
T: Future,
S: Schedule,
{
- let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new()));
+ let ptr = Box::into_raw(Cell::<_, S>::new(task, scheduler, State::new(), id));
let ptr = unsafe { NonNull::new_unchecked(ptr as *mut Header) };
RawTask { ptr }
@@ -57,19 +171,40 @@
RawTask { ptr }
}
- /// Returns a reference to the task's meta structure.
- ///
- /// Safe as `Header` is `Sync`.
+ pub(super) fn header_ptr(&self) -> NonNull<Header> {
+ self.ptr
+ }
+
+ pub(super) fn trailer_ptr(&self) -> NonNull<Trailer> {
+ unsafe { Header::get_trailer(self.ptr) }
+ }
+
+ /// Returns a reference to the task's header.
pub(super) fn header(&self) -> &Header {
unsafe { self.ptr.as_ref() }
}
+ /// Returns a reference to the task's trailer.
+ pub(super) fn trailer(&self) -> &Trailer {
+ unsafe { &*self.trailer_ptr().as_ptr() }
+ }
+
+ /// Returns a reference to the task's state.
+ pub(super) fn state(&self) -> &State {
+ &self.header().state
+ }
+
/// Safety: mutual exclusion is required to call this function.
pub(super) fn poll(self) {
let vtable = self.header().vtable;
unsafe { (vtable.poll)(self.ptr) }
}
+ pub(super) fn schedule(self) {
+ let vtable = self.header().vtable;
+ unsafe { (vtable.schedule)(self.ptr) }
+ }
+
pub(super) fn dealloc(self) {
let vtable = self.header().vtable;
unsafe {
@@ -89,14 +224,21 @@
unsafe { (vtable.drop_join_handle_slow)(self.ptr) }
}
+ pub(super) fn drop_abort_handle(self) {
+ let vtable = self.header().vtable;
+ unsafe { (vtable.drop_abort_handle)(self.ptr) }
+ }
+
pub(super) fn shutdown(self) {
let vtable = self.header().vtable;
unsafe { (vtable.shutdown)(self.ptr) }
}
- pub(super) fn remote_abort(self) {
- let vtable = self.header().vtable;
- unsafe { (vtable.remote_abort)(self.ptr) }
+ /// Increment the task's reference count.
+ ///
+ /// Currently, this is used only when creating an `AbortHandle`.
+ pub(super) fn ref_inc(self) {
+ self.header().state.ref_inc();
}
}
@@ -113,6 +255,15 @@
harness.poll();
}
+unsafe fn schedule<S: Schedule>(ptr: NonNull<Header>) {
+ use crate::runtime::task::{Notified, Task};
+
+ let scheduler = Header::get_scheduler::<S>(ptr);
+ scheduler
+ .as_ref()
+ .schedule(Notified(Task::from_raw(ptr.cast())));
+}
+
unsafe fn dealloc<T: Future, S: Schedule>(ptr: NonNull<Header>) {
let harness = Harness::<T, S>::from_raw(ptr);
harness.dealloc();
@@ -134,9 +285,9 @@
harness.drop_join_handle_slow()
}
-unsafe fn remote_abort<T: Future, S: Schedule>(ptr: NonNull<Header>) {
+unsafe fn drop_abort_handle<T: Future, S: Schedule>(ptr: NonNull<Header>) {
let harness = Harness::<T, S>::from_raw(ptr);
- harness.remote_abort()
+ harness.drop_reference();
}
unsafe fn shutdown<T: Future, S: Schedule>(ptr: NonNull<Header>) {
diff --git a/src/runtime/task/state.rs b/src/runtime/task/state.rs
index c2d5b28..7728312 100644
--- a/src/runtime/task/state.rs
+++ b/src/runtime/task/state.rs
@@ -378,7 +378,7 @@
pub(super) fn set_join_waker(&self) -> UpdateResult {
self.fetch_update(|curr| {
assert!(curr.is_join_interested());
- assert!(!curr.has_join_waker());
+ assert!(!curr.is_join_waker_set());
if curr.is_complete() {
return None;
@@ -398,7 +398,7 @@
pub(super) fn unset_waker(&self) -> UpdateResult {
self.fetch_update(|curr| {
assert!(curr.is_join_interested());
- assert!(curr.has_join_waker());
+ assert!(curr.is_join_waker_set());
if curr.is_complete() {
return None;
@@ -546,7 +546,7 @@
self.0 &= !JOIN_INTEREST
}
- pub(super) fn has_join_waker(self) -> bool {
+ pub(super) fn is_join_waker_set(self) -> bool {
self.0 & JOIN_WAKER == JOIN_WAKER
}
@@ -588,7 +588,7 @@
.field("is_notified", &self.is_notified())
.field("is_cancelled", &self.is_cancelled())
.field("is_join_interested", &self.is_join_interested())
- .field("has_join_waker", &self.has_join_waker())
+ .field("is_join_waker_set", &self.is_join_waker_set())
.field("ref_count", &self.ref_count())
.finish()
}
diff --git a/src/runtime/task/waker.rs b/src/runtime/task/waker.rs
index b7313b4..b5f5ace 100644
--- a/src/runtime/task/waker.rs
+++ b/src/runtime/task/waker.rs
@@ -1,6 +1,5 @@
use crate::future::Future;
-use crate::runtime::task::harness::Harness;
-use crate::runtime::task::{Header, Schedule};
+use crate::runtime::task::{Header, RawTask, Schedule};
use std::marker::PhantomData;
use std::mem::ManuallyDrop;
@@ -13,9 +12,9 @@
_p: PhantomData<(&'a Header, S)>,
}
-/// Returns a `WakerRef` which avoids having to pre-emptively increase the
+/// Returns a `WakerRef` which avoids having to preemptively increase the
/// refcount if there is no need to do so.
-pub(super) fn waker_ref<T, S>(header: &Header) -> WakerRef<'_, S>
+pub(super) fn waker_ref<T, S>(header: &NonNull<Header>) -> WakerRef<'_, S>
where
T: Future,
S: Schedule,
@@ -28,7 +27,7 @@
// point and not an *owned* waker, we must ensure that `drop` is never
// called on this waker instance. This is done by wrapping it with
// `ManuallyDrop` and then never calling drop.
- let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker::<T, S>(header))) };
+ let waker = unsafe { ManuallyDrop::new(Waker::from_raw(raw_waker(*header))) };
WakerRef {
waker,
@@ -46,8 +45,8 @@
cfg_trace! {
macro_rules! trace {
- ($harness:expr, $op:expr) => {
- if let Some(id) = $harness.id() {
+ ($header:expr, $op:expr) => {
+ if let Some(id) = Header::get_tracing_id(&$header) {
tracing::trace!(
target: "tokio::task::waker",
op = $op,
@@ -60,71 +59,46 @@
cfg_not_trace! {
macro_rules! trace {
- ($harness:expr, $op:expr) => {
+ ($header:expr, $op:expr) => {
// noop
- let _ = &$harness;
+ let _ = &$header;
}
}
}
-unsafe fn clone_waker<T, S>(ptr: *const ()) -> RawWaker
-where
- T: Future,
- S: Schedule,
-{
- let header = ptr as *const Header;
- let ptr = NonNull::new_unchecked(ptr as *mut Header);
- let harness = Harness::<T, S>::from_raw(ptr);
- trace!(harness, "waker.clone");
- (*header).state.ref_inc();
- raw_waker::<T, S>(header)
+unsafe fn clone_waker(ptr: *const ()) -> RawWaker {
+ let header = NonNull::new_unchecked(ptr as *mut Header);
+ trace!(header, "waker.clone");
+ header.as_ref().state.ref_inc();
+ raw_waker(header)
}
-unsafe fn drop_waker<T, S>(ptr: *const ())
-where
- T: Future,
- S: Schedule,
-{
+unsafe fn drop_waker(ptr: *const ()) {
let ptr = NonNull::new_unchecked(ptr as *mut Header);
- let harness = Harness::<T, S>::from_raw(ptr);
- trace!(harness, "waker.drop");
- harness.drop_reference();
+ trace!(ptr, "waker.drop");
+ let raw = RawTask::from_raw(ptr);
+ raw.drop_reference();
}
-unsafe fn wake_by_val<T, S>(ptr: *const ())
-where
- T: Future,
- S: Schedule,
-{
+unsafe fn wake_by_val(ptr: *const ()) {
let ptr = NonNull::new_unchecked(ptr as *mut Header);
- let harness = Harness::<T, S>::from_raw(ptr);
- trace!(harness, "waker.wake");
- harness.wake_by_val();
+ trace!(ptr, "waker.wake");
+ let raw = RawTask::from_raw(ptr);
+ raw.wake_by_val();
}
// Wake without consuming the waker
-unsafe fn wake_by_ref<T, S>(ptr: *const ())
-where
- T: Future,
- S: Schedule,
-{
+unsafe fn wake_by_ref(ptr: *const ()) {
let ptr = NonNull::new_unchecked(ptr as *mut Header);
- let harness = Harness::<T, S>::from_raw(ptr);
- trace!(harness, "waker.wake_by_ref");
- harness.wake_by_ref();
+ trace!(ptr, "waker.wake_by_ref");
+ let raw = RawTask::from_raw(ptr);
+ raw.wake_by_ref();
}
-fn raw_waker<T, S>(header: *const Header) -> RawWaker
-where
- T: Future,
- S: Schedule,
-{
- let ptr = header as *const ();
- let vtable = &RawWakerVTable::new(
- clone_waker::<T, S>,
- wake_by_val::<T, S>,
- wake_by_ref::<T, S>,
- drop_waker::<T, S>,
- );
- RawWaker::new(ptr, vtable)
+static WAKER_VTABLE: RawWakerVTable =
+ RawWakerVTable::new(clone_waker, wake_by_val, wake_by_ref, drop_waker);
+
+fn raw_waker(header: NonNull<Header>) -> RawWaker {
+ let ptr = header.as_ptr() as *const ();
+ RawWaker::new(ptr, &WAKER_VTABLE)
}
diff --git a/src/runtime/tests/loom_blocking.rs b/src/runtime/tests/loom_blocking.rs
index 8fb54c5..5c4aeae 100644
--- a/src/runtime/tests/loom_blocking.rs
+++ b/src/runtime/tests/loom_blocking.rs
@@ -23,6 +23,77 @@
});
}
+#[test]
+fn spawn_mandatory_blocking_should_always_run() {
+ use crate::runtime::tests::loom_oneshot;
+ loom::model(|| {
+ let rt = runtime::Builder::new_current_thread().build().unwrap();
+
+ let (tx, rx) = loom_oneshot::channel();
+ let _enter = rt.enter();
+ runtime::spawn_blocking(|| {});
+ runtime::spawn_mandatory_blocking(move || {
+ let _ = tx.send(());
+ })
+ .unwrap();
+
+ drop(rt);
+
+ // This call will deadlock if `spawn_mandatory_blocking` doesn't run.
+ let () = rx.recv();
+ });
+}
+
+#[test]
+fn spawn_mandatory_blocking_should_run_even_when_shutting_down_from_other_thread() {
+ use crate::runtime::tests::loom_oneshot;
+ loom::model(|| {
+ let rt = runtime::Builder::new_current_thread().build().unwrap();
+ let handle = rt.handle().clone();
+
+ // Drop the runtime in a different thread
+ {
+ loom::thread::spawn(move || {
+ drop(rt);
+ });
+ }
+
+ let _enter = handle.enter();
+ let (tx, rx) = loom_oneshot::channel();
+ let handle = runtime::spawn_mandatory_blocking(move || {
+ let _ = tx.send(());
+ });
+
+ // handle.is_some() means that `spawn_mandatory_blocking`
+ // promised us to run the blocking task
+ if handle.is_some() {
+ // This call will deadlock if `spawn_mandatory_blocking` doesn't run.
+ let () = rx.recv();
+ }
+ });
+}
+
+#[test]
+fn spawn_blocking_when_paused() {
+ use std::time::Duration;
+ loom::model(|| {
+ let rt = crate::runtime::Builder::new_current_thread()
+ .enable_time()
+ .start_paused(true)
+ .build()
+ .unwrap();
+ let handle = rt.handle();
+ let _enter = handle.enter();
+ let a = crate::task::spawn_blocking(|| {});
+ let b = crate::task::spawn_blocking(|| {});
+ rt.block_on(crate::time::timeout(Duration::from_millis(1), async move {
+ a.await.expect("blocking task should finish");
+ b.await.expect("blocking task should finish");
+ }))
+ .expect("timeout should not trigger");
+ });
+}
+
fn mk_runtime(num_threads: usize) -> Runtime {
runtime::Builder::new_multi_thread()
.worker_threads(num_threads)
diff --git a/src/runtime/tests/loom_basic_scheduler.rs b/src/runtime/tests/loom_current_thread_scheduler.rs
similarity index 86%
rename from src/runtime/tests/loom_basic_scheduler.rs
rename to src/runtime/tests/loom_current_thread_scheduler.rs
index d2894b9..a772603 100644
--- a/src/runtime/tests/loom_basic_scheduler.rs
+++ b/src/runtime/tests/loom_current_thread_scheduler.rs
@@ -34,20 +34,22 @@
#[test]
fn block_on_num_polls() {
loom::model(|| {
- // we expect at most 3 number of polls because there are
- // three points at which we poll the future. At any of these
- // points it can be ready:
+ // we expect at most 4 number of polls because there are three points at
+ // which we poll the future and an opportunity for a false-positive.. At
+ // any of these points it can be ready:
//
- // - when we fail to steal the parker and we block on a
- // notification that it is available.
+ // - when we fail to steal the parker and we block on a notification
+ // that it is available.
//
// - when we steal the parker and we schedule the future
//
- // - when the future is woken up and we have ran the max
- // number of tasks for the current tick or there are no
- // more tasks to run.
+ // - when the future is woken up and we have ran the max number of tasks
+ // for the current tick or there are no more tasks to run.
//
- let at_most = 3;
+ // - a thread is notified that the parker is available but a third
+ // thread acquires it before the notified thread can.
+ //
+ let at_most = 4;
let rt1 = Arc::new(Builder::new_current_thread().build().unwrap());
let rt2 = rt1.clone();
diff --git a/src/runtime/tests/loom_join_set.rs b/src/runtime/tests/loom_join_set.rs
new file mode 100644
index 0000000..bd34387
--- /dev/null
+++ b/src/runtime/tests/loom_join_set.rs
@@ -0,0 +1,82 @@
+use crate::runtime::Builder;
+use crate::task::JoinSet;
+
+#[test]
+fn test_join_set() {
+ loom::model(|| {
+ let rt = Builder::new_multi_thread()
+ .worker_threads(1)
+ .build()
+ .unwrap();
+ let mut set = JoinSet::new();
+
+ rt.block_on(async {
+ assert_eq!(set.len(), 0);
+ set.spawn(async { () });
+ assert_eq!(set.len(), 1);
+ set.spawn(async { () });
+ assert_eq!(set.len(), 2);
+ let () = set.join_next().await.unwrap().unwrap();
+ assert_eq!(set.len(), 1);
+ set.spawn(async { () });
+ assert_eq!(set.len(), 2);
+ let () = set.join_next().await.unwrap().unwrap();
+ assert_eq!(set.len(), 1);
+ let () = set.join_next().await.unwrap().unwrap();
+ assert_eq!(set.len(), 0);
+ set.spawn(async { () });
+ assert_eq!(set.len(), 1);
+ });
+
+ drop(set);
+ drop(rt);
+ });
+}
+
+#[test]
+fn abort_all_during_completion() {
+ use std::sync::{
+ atomic::{AtomicBool, Ordering::SeqCst},
+ Arc,
+ };
+
+ // These booleans assert that at least one execution had the task complete first, and that at
+ // least one execution had the task be cancelled before it completed.
+ let complete_happened = Arc::new(AtomicBool::new(false));
+ let cancel_happened = Arc::new(AtomicBool::new(false));
+
+ {
+ let complete_happened = complete_happened.clone();
+ let cancel_happened = cancel_happened.clone();
+ loom::model(move || {
+ let rt = Builder::new_multi_thread()
+ .worker_threads(1)
+ .build()
+ .unwrap();
+
+ let mut set = JoinSet::new();
+
+ rt.block_on(async {
+ set.spawn(async { () });
+ set.abort_all();
+
+ match set.join_next().await {
+ Some(Ok(())) => complete_happened.store(true, SeqCst),
+ Some(Err(err)) if err.is_cancelled() => cancel_happened.store(true, SeqCst),
+ Some(Err(err)) => panic!("fail: {}", err),
+ None => {
+ unreachable!("Aborting the task does not remove it from the JoinSet.")
+ }
+ }
+
+ assert!(matches!(set.join_next().await, None));
+ });
+
+ drop(set);
+ drop(rt);
+ });
+ }
+
+ assert!(complete_happened.load(SeqCst));
+ assert!(cancel_happened.load(SeqCst));
+}
diff --git a/src/runtime/tests/loom_queue.rs b/src/runtime/tests/loom_queue.rs
index 2cbb0a1..fc93bf3 100644
--- a/src/runtime/tests/loom_queue.rs
+++ b/src/runtime/tests/loom_queue.rs
@@ -1,7 +1,7 @@
-use crate::runtime::blocking::NoopSchedule;
-use crate::runtime::queue;
-use crate::runtime::stats::WorkerStatsBatcher;
+use crate::runtime::scheduler::multi_thread::queue;
use crate::runtime::task::Inject;
+use crate::runtime::tests::NoopSchedule;
+use crate::runtime::MetricsBatch;
use loom::thread;
@@ -10,14 +10,15 @@
loom::model(|| {
let (steal, mut local) = queue::local();
let inject = Inject::new();
+ let mut metrics = MetricsBatch::new();
let th = thread::spawn(move || {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (_, mut local) = queue::local();
let mut n = 0;
for _ in 0..3 {
- if steal.steal_into(&mut local, &mut stats).is_some() {
+ if steal.steal_into(&mut local, &mut metrics).is_some() {
n += 1;
}
@@ -34,7 +35,7 @@
for _ in 0..2 {
for _ in 0..2 {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
}
if local.pop().is_some() {
@@ -43,7 +44,7 @@
// Push another task
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
while local.pop().is_some() {
n += 1;
@@ -65,13 +66,14 @@
loom::model(|| {
let (steal, mut local) = queue::local();
let inject = Inject::new();
+ let mut metrics = MetricsBatch::new();
let th = thread::spawn(move || {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (_, mut local) = queue::local();
let mut n = 0;
- if steal.steal_into(&mut local, &mut stats).is_some() {
+ if steal.steal_into(&mut local, &mut metrics).is_some() {
n += 1;
}
@@ -86,7 +88,7 @@
// push a task, pop a task
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
if local.pop().is_some() {
n += 1;
@@ -94,7 +96,7 @@
for _ in 0..6 {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
}
n += th.join().unwrap();
@@ -116,10 +118,10 @@
const NUM_TASKS: usize = 5;
fn steal_tasks(steal: queue::Steal<NoopSchedule>) -> usize {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (_, mut local) = queue::local();
- if steal.steal_into(&mut local, &mut stats).is_none() {
+ if steal.steal_into(&mut local, &mut metrics).is_none() {
return 0;
}
@@ -135,11 +137,12 @@
loom::model(|| {
let (steal, mut local) = queue::local();
let inject = Inject::new();
+ let mut metrics = MetricsBatch::new();
// Push work
for _ in 0..NUM_TASKS {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
}
let th1 = {
@@ -169,7 +172,7 @@
#[test]
fn chained_steal() {
loom::model(|| {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (s1, mut l1) = queue::local();
let (s2, mut l2) = queue::local();
let inject = Inject::new();
@@ -177,17 +180,17 @@
// Load up some tasks
for _ in 0..4 {
let (task, _) = super::unowned(async {});
- l1.push_back(task, &inject);
+ l1.push_back(task, &inject, &mut metrics);
let (task, _) = super::unowned(async {});
- l2.push_back(task, &inject);
+ l2.push_back(task, &inject, &mut metrics);
}
// Spawn a task to steal from **our** queue
let th = thread::spawn(move || {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (_, mut local) = queue::local();
- s1.steal_into(&mut local, &mut stats);
+ s1.steal_into(&mut local, &mut metrics);
while local.pop().is_some() {}
});
@@ -195,7 +198,7 @@
// Drain our tasks, then attempt to steal
while l1.pop().is_some() {}
- s2.steal_into(&mut l1, &mut stats);
+ s2.steal_into(&mut l1, &mut metrics);
th.join().unwrap();
diff --git a/src/runtime/tests/loom_yield.rs b/src/runtime/tests/loom_yield.rs
new file mode 100644
index 0000000..ba506e5
--- /dev/null
+++ b/src/runtime/tests/loom_yield.rs
@@ -0,0 +1,37 @@
+use crate::runtime::park;
+use crate::runtime::tests::loom_oneshot as oneshot;
+use crate::runtime::{self, Runtime};
+
+#[test]
+fn yield_calls_park_before_scheduling_again() {
+ // Don't need to check all permutations
+ let mut loom = loom::model::Builder::default();
+ loom.max_permutations = Some(1);
+ loom.check(|| {
+ let rt = mk_runtime(2);
+ let (tx, rx) = oneshot::channel::<()>();
+
+ rt.spawn(async {
+ let tid = loom::thread::current().id();
+ let park_count = park::current_thread_park_count();
+
+ crate::task::yield_now().await;
+
+ if tid == loom::thread::current().id() {
+ let new_park_count = park::current_thread_park_count();
+ assert_eq!(park_count + 1, new_park_count);
+ }
+
+ tx.send(());
+ });
+
+ rx.recv();
+ });
+}
+
+fn mk_runtime(num_threads: usize) -> Runtime {
+ runtime::Builder::new_multi_thread()
+ .worker_threads(num_threads)
+ .build()
+ .unwrap()
+}
diff --git a/src/runtime/tests/mod.rs b/src/runtime/tests/mod.rs
index be36d6f..4e7c245 100644
--- a/src/runtime/tests/mod.rs
+++ b/src/runtime/tests/mod.rs
@@ -1,8 +1,30 @@
+// Enable dead_code / unreachable_pub here. It has been disabled in lib.rs for
+// other code when running loom tests.
+#![cfg_attr(loom, warn(dead_code, unreachable_pub))]
+
+use self::noop_scheduler::NoopSchedule;
use self::unowned_wrapper::unowned;
+mod noop_scheduler {
+ use crate::runtime::task::{self, Task};
+
+ /// `task::Schedule` implementation that does nothing, for testing.
+ pub(crate) struct NoopSchedule;
+
+ impl task::Schedule for NoopSchedule {
+ fn release(&self, _task: &Task<Self>) -> Option<Task<Self>> {
+ None
+ }
+
+ fn schedule(&self, _task: task::Notified<Self>) {
+ unreachable!();
+ }
+ }
+}
+
mod unowned_wrapper {
- use crate::runtime::blocking::NoopSchedule;
- use crate::runtime::task::{JoinHandle, Notified};
+ use crate::runtime::task::{Id, JoinHandle, Notified};
+ use crate::runtime::tests::NoopSchedule;
#[cfg(all(tokio_unstable, feature = "tracing"))]
pub(crate) fn unowned<T>(task: T) -> (Notified<NoopSchedule>, JoinHandle<T::Output>)
@@ -13,7 +35,7 @@
use tracing::Instrument;
let span = tracing::trace_span!("test_span");
let task = task.instrument(span);
- let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule);
+ let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule, Id::next());
(task.into_notified(), handle)
}
@@ -23,19 +45,21 @@
T: std::future::Future + Send + 'static,
T::Output: Send + 'static,
{
- let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule);
+ let (task, handle) = crate::runtime::task::unowned(task, NoopSchedule, Id::next());
(task.into_notified(), handle)
}
}
cfg_loom! {
- mod loom_basic_scheduler;
- mod loom_local;
mod loom_blocking;
+ mod loom_current_thread_scheduler;
+ mod loom_local;
mod loom_oneshot;
mod loom_pool;
mod loom_queue;
mod loom_shutdown_join;
+ mod loom_join_set;
+ mod loom_yield;
}
cfg_not_loom! {
diff --git a/src/runtime/tests/queue.rs b/src/runtime/tests/queue.rs
index 47f1b01..68d2e89 100644
--- a/src/runtime/tests/queue.rs
+++ b/src/runtime/tests/queue.rs
@@ -1,18 +1,39 @@
-use crate::runtime::queue;
-use crate::runtime::stats::WorkerStatsBatcher;
+use crate::runtime::scheduler::multi_thread::queue;
use crate::runtime::task::{self, Inject, Schedule, Task};
+use crate::runtime::MetricsBatch;
use std::thread;
use std::time::Duration;
+#[allow(unused)]
+macro_rules! assert_metrics {
+ ($metrics:ident, $field:ident == $v:expr) => {{
+ use crate::runtime::WorkerMetrics;
+ use std::sync::atomic::Ordering::Relaxed;
+
+ let worker = WorkerMetrics::new();
+ $metrics.submit(&worker);
+
+ let expect = $v;
+ let actual = worker.$field.load(Relaxed);
+
+ assert!(actual == expect, "expect = {}; actual = {}", expect, actual)
+ }};
+}
+
#[test]
fn fits_256() {
let (_, mut local) = queue::local();
let inject = Inject::new();
+ let mut metrics = MetricsBatch::new();
for _ in 0..256 {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
+ }
+
+ cfg_metrics! {
+ assert_metrics!(metrics, overflow_count == 0);
}
assert!(inject.pop().is_none());
@@ -24,10 +45,15 @@
fn overflow() {
let (_, mut local) = queue::local();
let inject = Inject::new();
+ let mut metrics = MetricsBatch::new();
for _ in 0..257 {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
+ }
+
+ cfg_metrics! {
+ assert_metrics!(metrics, overflow_count == 1);
}
let mut n = 0;
@@ -45,7 +71,7 @@
#[test]
fn steal_batch() {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (steal1, mut local1) = queue::local();
let (_, mut local2) = queue::local();
@@ -53,10 +79,14 @@
for _ in 0..4 {
let (task, _) = super::unowned(async {});
- local1.push_back(task, &inject);
+ local1.push_back(task, &inject, &mut metrics);
}
- assert!(steal1.steal_into(&mut local2, &mut stats).is_some());
+ assert!(steal1.steal_into(&mut local2, &mut metrics).is_some());
+
+ cfg_metrics! {
+ assert_metrics!(metrics, steal_count == 2);
+ }
for _ in 0..1 {
assert!(local2.pop().is_some());
@@ -71,25 +101,35 @@
assert!(local1.pop().is_none());
}
+const fn normal_or_miri(normal: usize, miri: usize) -> usize {
+ if cfg!(miri) {
+ miri
+ } else {
+ normal
+ }
+}
+
#[test]
fn stress1() {
- const NUM_ITER: usize = 1;
- const NUM_STEAL: usize = 1_000;
- const NUM_LOCAL: usize = 1_000;
- const NUM_PUSH: usize = 500;
- const NUM_POP: usize = 250;
+ const NUM_ITER: usize = 5;
+ const NUM_STEAL: usize = normal_or_miri(1_000, 10);
+ const NUM_LOCAL: usize = normal_or_miri(1_000, 10);
+ const NUM_PUSH: usize = normal_or_miri(500, 10);
+ const NUM_POP: usize = normal_or_miri(250, 10);
+
+ let mut metrics = MetricsBatch::new();
for _ in 0..NUM_ITER {
let (steal, mut local) = queue::local();
let inject = Inject::new();
let th = thread::spawn(move || {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut metrics = MetricsBatch::new();
let (_, mut local) = queue::local();
let mut n = 0;
for _ in 0..NUM_STEAL {
- if steal.steal_into(&mut local, &mut stats).is_some() {
+ if steal.steal_into(&mut local, &mut metrics).is_some() {
n += 1;
}
@@ -100,6 +140,10 @@
thread::yield_now();
}
+ cfg_metrics! {
+ assert_metrics!(metrics, steal_count == n as _);
+ }
+
n
});
@@ -108,7 +152,7 @@
for _ in 0..NUM_LOCAL {
for _ in 0..NUM_PUSH {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
}
for _ in 0..NUM_POP {
@@ -133,15 +177,17 @@
#[test]
fn stress2() {
const NUM_ITER: usize = 1;
- const NUM_TASKS: usize = 1_000_000;
- const NUM_STEAL: usize = 1_000;
+ const NUM_TASKS: usize = normal_or_miri(1_000_000, 50);
+ const NUM_STEAL: usize = normal_or_miri(1_000, 10);
+
+ let mut metrics = MetricsBatch::new();
for _ in 0..NUM_ITER {
let (steal, mut local) = queue::local();
let inject = Inject::new();
let th = thread::spawn(move || {
- let mut stats = WorkerStatsBatcher::new(0);
+ let mut stats = MetricsBatch::new();
let (_, mut local) = queue::local();
let mut n = 0;
@@ -164,7 +210,7 @@
for i in 0..NUM_TASKS {
let (task, _) = super::unowned(async {});
- local.push_back(task, &inject);
+ local.push_back(task, &inject, &mut metrics);
if i % 128 == 0 && local.pop().is_some() {
num_pop += 1;
diff --git a/src/runtime/tests/task.rs b/src/runtime/tests/task.rs
index 04e1b56..a79c0f5 100644
--- a/src/runtime/tests/task.rs
+++ b/src/runtime/tests/task.rs
@@ -1,5 +1,5 @@
-use crate::runtime::blocking::NoopSchedule;
-use crate::runtime::task::{self, unowned, JoinHandle, OwnedTasks, Schedule, Task};
+use crate::runtime::task::{self, unowned, Id, JoinHandle, OwnedTasks, Schedule, Task};
+use crate::runtime::tests::NoopSchedule;
use crate::util::TryLock;
use std::collections::VecDeque;
@@ -55,6 +55,7 @@
unreachable!()
},
NoopSchedule,
+ Id::next(),
);
drop(notified);
handle.assert_not_dropped();
@@ -71,6 +72,7 @@
unreachable!()
},
NoopSchedule,
+ Id::next(),
);
drop(join);
handle.assert_not_dropped();
@@ -78,6 +80,46 @@
handle.assert_dropped();
}
+#[test]
+fn drop_abort_handle1() {
+ let (ad, handle) = AssertDrop::new();
+ let (notified, join) = unowned(
+ async {
+ drop(ad);
+ unreachable!()
+ },
+ NoopSchedule,
+ Id::next(),
+ );
+ let abort = join.abort_handle();
+ drop(join);
+ handle.assert_not_dropped();
+ drop(notified);
+ handle.assert_not_dropped();
+ drop(abort);
+ handle.assert_dropped();
+}
+
+#[test]
+fn drop_abort_handle2() {
+ let (ad, handle) = AssertDrop::new();
+ let (notified, join) = unowned(
+ async {
+ drop(ad);
+ unreachable!()
+ },
+ NoopSchedule,
+ Id::next(),
+ );
+ let abort = join.abort_handle();
+ drop(notified);
+ handle.assert_not_dropped();
+ drop(abort);
+ handle.assert_not_dropped();
+ drop(join);
+ handle.assert_dropped();
+}
+
// Shutting down through Notified works
#[test]
fn create_shutdown1() {
@@ -88,6 +130,7 @@
unreachable!()
},
NoopSchedule,
+ Id::next(),
);
drop(join);
handle.assert_not_dropped();
@@ -104,6 +147,7 @@
unreachable!()
},
NoopSchedule,
+ Id::next(),
);
handle.assert_not_dropped();
notified.shutdown();
@@ -113,7 +157,7 @@
#[test]
fn unowned_poll() {
- let (task, _) = unowned(async {}, NoopSchedule);
+ let (task, _) = unowned(async {}, NoopSchedule, Id::next());
task.run();
}
@@ -228,7 +272,7 @@
T: 'static + Send + Future,
T::Output: 'static + Send,
{
- let (handle, notified) = self.0.owned.bind(future, self.clone());
+ let (handle, notified) = self.0.owned.bind(future, self.clone(), Id::next());
if let Some(notified) = notified {
self.schedule(notified);
diff --git a/src/runtime/tests/task_combinations.rs b/src/runtime/tests/task_combinations.rs
index 76ce233..73a20d9 100644
--- a/src/runtime/tests/task_combinations.rs
+++ b/src/runtime/tests/task_combinations.rs
@@ -1,8 +1,10 @@
+use std::fmt;
use std::future::Future;
use std::panic;
use std::pin::Pin;
use std::task::{Context, Poll};
+use crate::runtime::task::AbortHandle;
use crate::runtime::Builder;
use crate::sync::oneshot;
use crate::task::JoinHandle;
@@ -56,6 +58,12 @@
AbortedAfterConsumeOutput = 4,
}
+#[derive(Copy, Clone, Debug, PartialEq)]
+enum CombiAbortSource {
+ JoinHandle,
+ AbortHandle,
+}
+
#[test]
fn test_combinations() {
let mut rt = &[
@@ -90,6 +98,13 @@
CombiAbort::AbortedAfterFinish,
CombiAbort::AbortedAfterConsumeOutput,
];
+ let ah = [
+ None,
+ Some(CombiJoinHandle::DropImmediately),
+ Some(CombiJoinHandle::DropFirstPoll),
+ Some(CombiJoinHandle::DropAfterNoConsume),
+ Some(CombiJoinHandle::DropAfterConsume),
+ ];
for rt in rt.iter().copied() {
for ls in ls.iter().copied() {
@@ -98,7 +113,34 @@
for ji in ji.iter().copied() {
for jh in jh.iter().copied() {
for abort in abort.iter().copied() {
- test_combination(rt, ls, task, output, ji, jh, abort);
+ // abort via join handle --- abort handles
+ // may be dropped at any point
+ for ah in ah.iter().copied() {
+ test_combination(
+ rt,
+ ls,
+ task,
+ output,
+ ji,
+ jh,
+ ah,
+ abort,
+ CombiAbortSource::JoinHandle,
+ );
+ }
+ // if aborting via AbortHandle, it will
+ // never be dropped.
+ test_combination(
+ rt,
+ ls,
+ task,
+ output,
+ ji,
+ jh,
+ None,
+ abort,
+ CombiAbortSource::AbortHandle,
+ );
}
}
}
@@ -108,6 +150,9 @@
}
}
+fn is_debug<T: fmt::Debug>(_: &T) {}
+
+#[allow(clippy::too_many_arguments)]
fn test_combination(
rt: CombiRuntime,
ls: CombiLocalSet,
@@ -115,12 +160,24 @@
output: CombiOutput,
ji: CombiJoinInterest,
jh: CombiJoinHandle,
+ ah: Option<CombiJoinHandle>,
abort: CombiAbort,
+ abort_src: CombiAbortSource,
) {
- if (jh as usize) < (abort as usize) {
- // drop before abort not possible
- return;
+ match (abort_src, ah) {
+ (CombiAbortSource::JoinHandle, _) if (jh as usize) < (abort as usize) => {
+ // join handle dropped prior to abort
+ return;
+ }
+ (CombiAbortSource::AbortHandle, Some(_)) => {
+ // abort handle dropped, we can't abort through the
+ // abort handle
+ return;
+ }
+
+ _ => {}
}
+
if (task == CombiTask::PanicOnDrop) && (output == CombiOutput::PanicOnDrop) {
// this causes double panic
return;
@@ -130,7 +187,15 @@
return;
}
- println!("Runtime {:?}, LocalSet {:?}, Task {:?}, Output {:?}, JoinInterest {:?}, JoinHandle {:?}, Abort {:?}", rt, ls, task, output, ji, jh, abort);
+ is_debug(&rt);
+ is_debug(&ls);
+ is_debug(&task);
+ is_debug(&output);
+ is_debug(&ji);
+ is_debug(&jh);
+ is_debug(&ah);
+ is_debug(&abort);
+ is_debug(&abort_src);
// A runtime optionally with a LocalSet
struct Rt {
@@ -282,8 +347,24 @@
);
}
+ // If we are either aborting the task via an abort handle, or dropping via
+ // an abort handle, do that now.
+ let mut abort_handle = if ah.is_some() || abort_src == CombiAbortSource::AbortHandle {
+ handle.as_ref().map(JoinHandle::abort_handle)
+ } else {
+ None
+ };
+
+ let do_abort = |abort_handle: &mut Option<AbortHandle>,
+ join_handle: Option<&mut JoinHandle<_>>| {
+ match abort_src {
+ CombiAbortSource::AbortHandle => abort_handle.take().unwrap().abort(),
+ CombiAbortSource::JoinHandle => join_handle.unwrap().abort(),
+ }
+ };
+
if abort == CombiAbort::AbortedImmediately {
- handle.as_mut().unwrap().abort();
+ do_abort(&mut abort_handle, handle.as_mut());
aborted = true;
}
if jh == CombiJoinHandle::DropImmediately {
@@ -301,12 +382,15 @@
}
if abort == CombiAbort::AbortedFirstPoll {
- handle.as_mut().unwrap().abort();
+ do_abort(&mut abort_handle, handle.as_mut());
aborted = true;
}
if jh == CombiJoinHandle::DropFirstPoll {
drop(handle.take().unwrap());
}
+ if ah == Some(CombiJoinHandle::DropFirstPoll) {
+ drop(abort_handle.take().unwrap());
+ }
// Signal the future that it can return now
let _ = on_complete.send(());
@@ -318,23 +402,42 @@
if abort == CombiAbort::AbortedAfterFinish {
// Don't set aborted to true here as the task already finished
- handle.as_mut().unwrap().abort();
+ do_abort(&mut abort_handle, handle.as_mut());
}
if jh == CombiJoinHandle::DropAfterNoConsume {
- // The runtime will usually have dropped every ref-count at this point,
- // in which case dropping the JoinHandle drops the output.
- //
- // (But it might race and still hold a ref-count)
- let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+ if ah == Some(CombiJoinHandle::DropAfterNoConsume) {
drop(handle.take().unwrap());
- }));
- if panic.is_err() {
- assert!(
- (output == CombiOutput::PanicOnDrop)
- && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop))
- && !aborted,
- "Dropping JoinHandle shouldn't panic here"
- );
+ // The runtime will usually have dropped every ref-count at this point,
+ // in which case dropping the AbortHandle drops the output.
+ //
+ // (But it might race and still hold a ref-count)
+ let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+ drop(abort_handle.take().unwrap());
+ }));
+ if panic.is_err() {
+ assert!(
+ (output == CombiOutput::PanicOnDrop)
+ && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop))
+ && !aborted,
+ "Dropping AbortHandle shouldn't panic here"
+ );
+ }
+ } else {
+ // The runtime will usually have dropped every ref-count at this point,
+ // in which case dropping the JoinHandle drops the output.
+ //
+ // (But it might race and still hold a ref-count)
+ let panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
+ drop(handle.take().unwrap());
+ }));
+ if panic.is_err() {
+ assert!(
+ (output == CombiOutput::PanicOnDrop)
+ && (!matches!(task, CombiTask::PanicOnRun | CombiTask::PanicOnRunAndDrop))
+ && !aborted,
+ "Dropping JoinHandle shouldn't panic here"
+ );
+ }
}
}
@@ -362,11 +465,15 @@
_ => unreachable!(),
}
- let handle = handle.take().unwrap();
+ let mut handle = handle.take().unwrap();
if abort == CombiAbort::AbortedAfterConsumeOutput {
- handle.abort();
+ do_abort(&mut abort_handle, Some(&mut handle));
}
drop(handle);
+
+ if ah == Some(CombiJoinHandle::DropAfterConsume) {
+ drop(abort_handle.take());
+ }
}
// The output should have been dropped now. Check whether the output
diff --git a/src/runtime/thread_id.rs b/src/runtime/thread_id.rs
new file mode 100644
index 0000000..ef39289
--- /dev/null
+++ b/src/runtime/thread_id.rs
@@ -0,0 +1,31 @@
+use std::num::NonZeroU64;
+
+#[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)]
+pub(crate) struct ThreadId(NonZeroU64);
+
+impl ThreadId {
+ pub(crate) fn next() -> Self {
+ use crate::loom::sync::atomic::{Ordering::Relaxed, StaticAtomicU64};
+
+ static NEXT_ID: StaticAtomicU64 = StaticAtomicU64::new(0);
+
+ let mut last = NEXT_ID.load(Relaxed);
+ loop {
+ let id = match last.checked_add(1) {
+ Some(id) => id,
+ None => exhausted(),
+ };
+
+ match NEXT_ID.compare_exchange_weak(last, id, Relaxed, Relaxed) {
+ Ok(_) => return ThreadId(NonZeroU64::new(id).unwrap()),
+ Err(id) => last = id,
+ }
+ }
+ }
+}
+
+#[cold]
+#[allow(dead_code)]
+fn exhausted() -> ! {
+ panic!("failed to generate unique thread ID: bitspace exhausted")
+}
diff --git a/src/runtime/thread_pool/mod.rs b/src/runtime/thread_pool/mod.rs
deleted file mode 100644
index 82e34c7..0000000
--- a/src/runtime/thread_pool/mod.rs
+++ /dev/null
@@ -1,118 +0,0 @@
-//! Threadpool
-
-mod atomic_cell;
-use atomic_cell::AtomicCell;
-
-mod idle;
-use self::idle::Idle;
-
-mod worker;
-pub(crate) use worker::Launch;
-
-pub(crate) use worker::block_in_place;
-
-use crate::loom::sync::Arc;
-use crate::runtime::stats::RuntimeStats;
-use crate::runtime::task::JoinHandle;
-use crate::runtime::{Callback, Parker};
-
-use std::fmt;
-use std::future::Future;
-
-/// Work-stealing based thread pool for executing futures.
-pub(crate) struct ThreadPool {
- spawner: Spawner,
-}
-
-/// Submits futures to the associated thread pool for execution.
-///
-/// A `Spawner` instance is a handle to a single thread pool that allows the owner
-/// of the handle to spawn futures onto the thread pool.
-///
-/// The `Spawner` handle is *only* used for spawning new futures. It does not
-/// impact the lifecycle of the thread pool in any way. The thread pool may
-/// shut down while there are outstanding `Spawner` instances.
-///
-/// `Spawner` instances are obtained by calling [`ThreadPool::spawner`].
-///
-/// [`ThreadPool::spawner`]: method@ThreadPool::spawner
-#[derive(Clone)]
-pub(crate) struct Spawner {
- shared: Arc<worker::Shared>,
-}
-
-// ===== impl ThreadPool =====
-
-impl ThreadPool {
- pub(crate) fn new(
- size: usize,
- parker: Parker,
- before_park: Option<Callback>,
- after_unpark: Option<Callback>,
- ) -> (ThreadPool, Launch) {
- let (shared, launch) = worker::create(size, parker, before_park, after_unpark);
- let spawner = Spawner { shared };
- let thread_pool = ThreadPool { spawner };
-
- (thread_pool, launch)
- }
-
- /// Returns reference to `Spawner`.
- ///
- /// The `Spawner` handle can be cloned and enables spawning tasks from other
- /// threads.
- pub(crate) fn spawner(&self) -> &Spawner {
- &self.spawner
- }
-
- /// Blocks the current thread waiting for the future to complete.
- ///
- /// The future will execute on the current thread, but all spawned tasks
- /// will be executed on the thread pool.
- pub(crate) fn block_on<F>(&self, future: F) -> F::Output
- where
- F: Future,
- {
- let mut enter = crate::runtime::enter(true);
- enter.block_on(future).expect("failed to park thread")
- }
-}
-
-impl fmt::Debug for ThreadPool {
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt.debug_struct("ThreadPool").finish()
- }
-}
-
-impl Drop for ThreadPool {
- fn drop(&mut self) {
- self.spawner.shutdown();
- }
-}
-
-// ==== impl Spawner =====
-
-impl Spawner {
- /// Spawns a future onto the thread pool
- pub(crate) fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
- where
- F: crate::future::Future + Send + 'static,
- F::Output: Send + 'static,
- {
- worker::Shared::bind_new_task(&self.shared, future)
- }
-
- pub(crate) fn shutdown(&mut self) {
- self.shared.close();
- }
-
- pub(crate) fn stats(&self) -> &RuntimeStats {
- self.shared.stats()
- }
-}
-
-impl fmt::Debug for Spawner {
- fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt.debug_struct("Spawner").finish()
- }
-}
diff --git a/src/time/driver/entry.rs b/src/runtime/time/entry.rs
similarity index 82%
rename from src/time/driver/entry.rs
rename to src/runtime/time/entry.rs
index 9e9f0dc..f0d613a 100644
--- a/src/time/driver/entry.rs
+++ b/src/runtime/time/entry.rs
@@ -5,9 +5,9 @@
//!
//! # Ground rules
//!
-//! The heart of the timer implementation here is the `TimerShared` structure,
-//! shared between the `TimerEntry` and the driver. Generally, we permit access
-//! to `TimerShared` ONLY via either 1) a mutable reference to `TimerEntry` or
+//! The heart of the timer implementation here is the [`TimerShared`] structure,
+//! shared between the [`TimerEntry`] and the driver. Generally, we permit access
+//! to [`TimerShared`] ONLY via either 1) a mutable reference to [`TimerEntry`] or
//! 2) a held driver lock.
//!
//! It follows from this that any changes made while holding BOTH 1 and 2 will
@@ -36,7 +36,7 @@
//! point than it was originally scheduled for.
//!
//! This is accomplished by lazily rescheduling timers. That is, we update the
-//! state field field with the true expiration of the timer from the holder of
+//! state field with the true expiration of the timer from the holder of
//! the [`TimerEntry`]. When the driver services timers (ie, whenever it's
//! walking lists of timers), it checks this "true when" value, and reschedules
//! based on it.
@@ -49,19 +49,20 @@
//! There is of course a race condition between timer reset and timer
//! expiration. If the driver fails to observe the updated expiration time, it
//! could trigger expiration of the timer too early. However, because
-//! `mark_pending` performs a compare-and-swap, it will identify this race and
+//! [`mark_pending`][mark_pending] performs a compare-and-swap, it will identify this race and
//! refuse to mark the timer as pending.
+//!
+//! [mark_pending]: TimerHandle::mark_pending
use crate::loom::cell::UnsafeCell;
use crate::loom::sync::atomic::AtomicU64;
use crate::loom::sync::atomic::Ordering;
+use crate::runtime::scheduler;
use crate::sync::AtomicWaker;
use crate::time::Instant;
use crate::util::linked_list;
-use super::Handle;
-
use std::cell::UnsafeCell as StdUnsafeCell;
use std::task::{Context, Poll, Waker};
use std::{marker::PhantomPinned, pin::Pin, ptr::NonNull};
@@ -171,7 +172,12 @@
let mut cur_state = self.state.load(Ordering::Relaxed);
loop {
- debug_assert!(cur_state < STATE_MIN_VALUE);
+ // improve the error message for things like
+ // https://github.com/tokio-rs/tokio/issues/3675
+ assert!(
+ cur_state < STATE_MIN_VALUE,
+ "mark_pending called when the timer entry is in an invalid state"
+ );
if cur_state > not_after {
break Err(cur_state);
@@ -281,10 +287,10 @@
/// timer. As this participates in intrusive data structures, it must be pinned
/// before polling.
#[derive(Debug)]
-pub(super) struct TimerEntry {
- /// Arc reference to the driver. We can only free the driver after
+pub(crate) struct TimerEntry {
+ /// Arc reference to the runtime handle. We can only free the driver after
/// deregistering everything from their respective timer wheels.
- driver: Handle,
+ driver: scheduler::Handle,
/// Shared inner structure; this is part of an intrusive linked list, and
/// therefore other references can exist to it while mutable references to
/// Entry exist.
@@ -324,18 +330,27 @@
///
/// Note that this structure is located inside the `TimerEntry` structure.
#[derive(Debug)]
+#[repr(C)]
pub(crate) struct TimerShared {
+ /// Data manipulated by the driver thread itself, only.
+ driver_state: CachePadded<TimerSharedPadded>,
+
/// Current state. This records whether the timer entry is currently under
/// the ownership of the driver, and if not, its current state (not
/// complete, fired, error, etc).
state: StateCell,
- /// Data manipulated by the driver thread itself, only.
- driver_state: CachePadded<TimerSharedPadded>,
-
_p: PhantomPinned,
}
+generate_addr_of_methods! {
+ impl<> TimerShared {
+ unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<TimerShared>> {
+ &self.driver_state.0.pointers
+ }
+ }
+}
+
impl TimerShared {
pub(super) fn new() -> Self {
Self {
@@ -419,6 +434,12 @@
/// frequently to minimize contention. In particular, we move it away from the
/// waker, as the waker is updated on every poll.
struct TimerSharedPadded {
+ /// A link within the doubly-linked list of timers on a particular level and
+ /// slot. Valid only if state is equal to Registered.
+ ///
+ /// Only accessed under the entry lock.
+ pointers: linked_list::Pointers<TimerShared>,
+
/// The expiration time for which this entry is currently registered.
/// Generally owned by the driver, but is accessed by the entry when not
/// registered.
@@ -426,12 +447,6 @@
/// The true expiration time. Set by the timer future, read by the driver.
true_when: AtomicU64,
-
- /// A link within the doubly-linked list of timers on a particular level and
- /// slot. Valid only if state is equal to Registered.
- ///
- /// Only accessed under the entry lock.
- pointers: StdUnsafeCell<linked_list::Pointers<TimerShared>>,
}
impl std::fmt::Debug for TimerSharedPadded {
@@ -448,7 +463,7 @@
Self {
cached_when: AtomicU64::new(0),
true_when: AtomicU64::new(0),
- pointers: StdUnsafeCell::new(linked_list::Pointers::new()),
+ pointers: linked_list::Pointers::new(),
}
}
}
@@ -472,14 +487,18 @@
unsafe fn pointers(
target: NonNull<Self::Target>,
) -> NonNull<linked_list::Pointers<Self::Target>> {
- unsafe { NonNull::new(target.as_ref().driver_state.0.pointers.get()).unwrap() }
+ TimerShared::addr_of_pointers(target)
}
}
// ===== impl Entry =====
impl TimerEntry {
- pub(crate) fn new(handle: &Handle, deadline: Instant) -> Self {
+ #[track_caller]
+ pub(crate) fn new(handle: &scheduler::Handle, deadline: Instant) -> Self {
+ // Panic if the time driver is not enabled
+ let _ = handle.driver().time();
+
let driver = handle.clone();
Self {
@@ -522,20 +541,21 @@
// driver did so far and happens-before everything the driver does in
// the future. While we have the lock held, we also go ahead and
// deregister the entry if necessary.
- unsafe { self.driver.clear_entry(NonNull::from(self.inner())) };
+ unsafe { self.driver().clear_entry(NonNull::from(self.inner())) };
}
pub(crate) fn reset(mut self: Pin<&mut Self>, new_time: Instant) {
unsafe { self.as_mut().get_unchecked_mut() }.initial_deadline = None;
- let tick = self.driver.time_source().deadline_to_tick(new_time);
+ let tick = self.driver().time_source().deadline_to_tick(new_time);
if self.inner().extend_expiration(tick).is_ok() {
return;
}
unsafe {
- self.driver.reregister(tick, self.inner().into());
+ self.driver()
+ .reregister(&self.driver.driver().io, tick, self.inner().into());
}
}
@@ -543,7 +563,7 @@
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Result<(), super::Error>> {
- if self.driver.is_shutdown() {
+ if self.driver().is_shutdown() {
panic!("{}", crate::util::error::RUNTIME_SHUTTING_DOWN_ERROR);
}
@@ -555,6 +575,10 @@
this.inner().state.poll(cx.waker())
}
+
+ pub(crate) fn driver(&self) -> &super::Handle {
+ self.driver.driver().time()
+ }
}
impl TimerHandle {
@@ -623,7 +647,73 @@
}
}
-#[cfg_attr(target_arch = "x86_64", repr(align(128)))]
-#[cfg_attr(not(target_arch = "x86_64"), repr(align(64)))]
+// Copied from [crossbeam/cache_padded](https://github.com/crossbeam-rs/crossbeam/blob/fa35346b7c789bba045ad789e894c68c466d1779/crossbeam-utils/src/cache_padded.rs#L62-L127)
+//
+// Starting from Intel's Sandy Bridge, spatial prefetcher is now pulling pairs of 64-byte cache
+// lines at a time, so we have to align to 128 bytes rather than 64.
+//
+// Sources:
+// - https://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-optimization-manual.pdf
+// - https://github.com/facebook/folly/blob/1b5288e6eea6df074758f877c849b6e73bbb9fbb/folly/lang/Align.h#L107
+//
+// ARM's big.LITTLE architecture has asymmetric cores and "big" cores have 128-byte cache line size.
+//
+// Sources:
+// - https://www.mono-project.com/news/2016/09/12/arm64-icache/
+//
+// powerpc64 has 128-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_ppc64x.go#L9
+#[cfg_attr(
+ any(
+ target_arch = "x86_64",
+ target_arch = "aarch64",
+ target_arch = "powerpc64",
+ ),
+ repr(align(128))
+)]
+// arm, mips, mips64, and riscv64 have 32-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_arm.go#L7
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips.go#L7
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mipsle.go#L7
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_mips64x.go#L9
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_riscv64.go#L7
+#[cfg_attr(
+ any(
+ target_arch = "arm",
+ target_arch = "mips",
+ target_arch = "mips64",
+ target_arch = "riscv64",
+ ),
+ repr(align(32))
+)]
+// s390x has 256-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_s390x.go#L7
+#[cfg_attr(target_arch = "s390x", repr(align(256)))]
+// x86 and wasm have 64-byte cache line size.
+//
+// Sources:
+// - https://github.com/golang/go/blob/dda2991c2ea0c5914714469c4defc2562a907230/src/internal/cpu/cpu_x86.go#L9
+// - https://github.com/golang/go/blob/3dd58676054223962cd915bb0934d1f9f489d4d2/src/internal/cpu/cpu_wasm.go#L7
+//
+// All others are assumed to have 64-byte cache line size.
+#[cfg_attr(
+ not(any(
+ target_arch = "x86_64",
+ target_arch = "aarch64",
+ target_arch = "powerpc64",
+ target_arch = "arm",
+ target_arch = "mips",
+ target_arch = "mips64",
+ target_arch = "riscv64",
+ target_arch = "s390x",
+ )),
+ repr(align(64))
+)]
#[derive(Debug, Default)]
struct CachePadded<T>(T);
diff --git a/src/runtime/time/handle.rs b/src/runtime/time/handle.rs
new file mode 100644
index 0000000..fce791d
--- /dev/null
+++ b/src/runtime/time/handle.rs
@@ -0,0 +1,62 @@
+use crate::runtime::time::TimeSource;
+use std::fmt;
+
+/// Handle to time driver instance.
+pub(crate) struct Handle {
+ pub(super) time_source: TimeSource,
+ pub(super) inner: super::Inner,
+}
+
+impl Handle {
+ /// Returns the time source associated with this handle.
+ pub(crate) fn time_source(&self) -> &TimeSource {
+ &self.time_source
+ }
+
+ /// Checks whether the driver has been shutdown.
+ pub(super) fn is_shutdown(&self) -> bool {
+ self.inner.is_shutdown()
+ }
+
+ /// Track that the driver is being unparked
+ pub(crate) fn unpark(&self) {
+ #[cfg(feature = "test-util")]
+ self.inner
+ .did_wake
+ .store(true, std::sync::atomic::Ordering::SeqCst);
+ }
+}
+
+cfg_not_rt! {
+ impl Handle {
+ /// Tries to get a handle to the current timer.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if there is no current timer set.
+ ///
+ /// It can be triggered when [`Builder::enable_time`] or
+ /// [`Builder::enable_all`] are not included in the builder.
+ ///
+ /// It can also panic whenever a timer is created outside of a
+ /// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic,
+ /// since the function is executed outside of the runtime.
+ /// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic.
+ /// And this is because wrapping the function on an async makes it lazy,
+ /// and so gets executed inside the runtime successfully without
+ /// panicking.
+ ///
+ /// [`Builder::enable_time`]: crate::runtime::Builder::enable_time
+ /// [`Builder::enable_all`]: crate::runtime::Builder::enable_all
+ #[track_caller]
+ pub(crate) fn current() -> Self {
+ panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR)
+ }
+ }
+}
+
+impl fmt::Debug for Handle {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "Handle")
+ }
+}
diff --git a/src/time/driver/mod.rs b/src/runtime/time/mod.rs
similarity index 66%
rename from src/time/driver/mod.rs
rename to src/runtime/time/mod.rs
index cf2290b..f81cab8 100644
--- a/src/time/driver/mod.rs
+++ b/src/runtime/time/mod.rs
@@ -7,22 +7,23 @@
//! Time driver.
mod entry;
-pub(self) use self::entry::{EntryList, TimerEntry, TimerHandle, TimerShared};
+pub(crate) use entry::TimerEntry;
+use entry::{EntryList, TimerHandle, TimerShared};
mod handle;
pub(crate) use self::handle::Handle;
+mod source;
+pub(crate) use source::TimeSource;
+
mod wheel;
-pub(super) mod sleep;
-
use crate::loom::sync::atomic::{AtomicBool, Ordering};
-use crate::loom::sync::{Arc, Mutex};
-use crate::park::{Park, Unpark};
+use crate::loom::sync::Mutex;
+use crate::runtime::driver::{self, IoHandle, IoStack};
use crate::time::error::Error;
-use crate::time::{Clock, Duration, Instant};
+use crate::time::{Clock, Duration};
-use std::convert::TryInto;
use std::fmt;
use std::{num::NonZeroU64, ptr::NonNull, task::Waker};
@@ -82,63 +83,9 @@
/// [timeout]: crate::time::Timeout
/// [interval]: crate::time::Interval
#[derive(Debug)]
-pub(crate) struct Driver<P: Park + 'static> {
- /// Timing backend in use.
- time_source: ClockTime,
-
- /// Shared state.
- handle: Handle,
-
+pub(crate) struct Driver {
/// Parker to delegate to.
- park: P,
-
- // When `true`, a call to `park_timeout` should immediately return and time
- // should not advance. One reason for this to be `true` is if the task
- // passed to `Runtime::block_on` called `task::yield_now()`.
- //
- // While it may look racy, it only has any effect when the clock is paused
- // and pausing the clock is restricted to a single-threaded runtime.
- #[cfg(feature = "test-util")]
- did_wake: Arc<AtomicBool>,
-}
-
-/// A structure which handles conversion from Instants to u64 timestamps.
-#[derive(Debug, Clone)]
-pub(self) struct ClockTime {
- clock: super::clock::Clock,
- start_time: Instant,
-}
-
-impl ClockTime {
- pub(self) fn new(clock: Clock) -> Self {
- Self {
- start_time: clock.now(),
- clock,
- }
- }
-
- pub(self) fn deadline_to_tick(&self, t: Instant) -> u64 {
- // Round up to the end of a ms
- self.instant_to_tick(t + Duration::from_nanos(999_999))
- }
-
- pub(self) fn instant_to_tick(&self, t: Instant) -> u64 {
- // round up
- let dur: Duration = t
- .checked_duration_since(self.start_time)
- .unwrap_or_else(|| Duration::from_secs(0));
- let ms = dur.as_millis();
-
- ms.try_into().expect("Duration too far into the future")
- }
-
- pub(self) fn tick_to_duration(&self, t: u64) -> Duration {
- Duration::from_millis(t)
- }
-
- pub(self) fn now(&self) -> u64 {
- self.instant_to_tick(self.clock.now())
- }
+ park: IoStack,
}
/// Timer state shared between `Driver`, `Handle`, and `Registration`.
@@ -148,13 +95,19 @@
/// True if the driver is being shutdown.
pub(super) is_shutdown: AtomicBool,
+
+ // When `true`, a call to `park_timeout` should immediately return and time
+ // should not advance. One reason for this to be `true` is if the task
+ // passed to `Runtime::block_on` called `task::yield_now()`.
+ //
+ // While it may look racy, it only has any effect when the clock is paused
+ // and pausing the clock is restricted to a single-threaded runtime.
+ #[cfg(feature = "test-util")]
+ did_wake: AtomicBool,
}
/// Time state shared which must be protected by a `Mutex`
struct InnerState {
- /// Timing backend in use.
- time_source: ClockTime,
-
/// The last published timer `elapsed` value.
elapsed: u64,
@@ -163,49 +116,67 @@
/// Timer wheel.
wheel: wheel::Wheel,
-
- /// Unparker that can be used to wake the time driver.
- unpark: Box<dyn Unpark>,
}
// ===== impl Driver =====
-impl<P> Driver<P>
-where
- P: Park + 'static,
-{
+impl Driver {
/// Creates a new `Driver` instance that uses `park` to block the current
/// thread and `time_source` to get the current time and convert to ticks.
///
/// Specifying the source of time is useful when testing.
- pub(crate) fn new(park: P, clock: Clock) -> Driver<P> {
- let time_source = ClockTime::new(clock);
+ pub(crate) fn new(park: IoStack, clock: Clock) -> (Driver, Handle) {
+ let time_source = TimeSource::new(clock);
- let inner = Inner::new(time_source.clone(), Box::new(park.unpark()));
-
- Driver {
+ let handle = Handle {
time_source,
- handle: Handle::new(Arc::new(inner)),
- park,
- #[cfg(feature = "test-util")]
- did_wake: Arc::new(AtomicBool::new(false)),
+ inner: Inner {
+ state: Mutex::new(InnerState {
+ elapsed: 0,
+ next_wake: None,
+ wheel: wheel::Wheel::new(),
+ }),
+ is_shutdown: AtomicBool::new(false),
+
+ #[cfg(feature = "test-util")]
+ did_wake: AtomicBool::new(false),
+ },
+ };
+
+ let driver = Driver { park };
+
+ (driver, handle)
+ }
+
+ pub(crate) fn park(&mut self, handle: &driver::Handle) {
+ self.park_internal(handle, None)
+ }
+
+ pub(crate) fn park_timeout(&mut self, handle: &driver::Handle, duration: Duration) {
+ self.park_internal(handle, Some(duration))
+ }
+
+ pub(crate) fn shutdown(&mut self, rt_handle: &driver::Handle) {
+ let handle = rt_handle.time();
+
+ if handle.is_shutdown() {
+ return;
}
+
+ handle.inner.is_shutdown.store(true, Ordering::SeqCst);
+
+ // Advance time forward to the end of time.
+
+ handle.process_at_time(u64::MAX);
+
+ self.park.shutdown(rt_handle);
}
- /// Returns a handle to the timer.
- ///
- /// The `Handle` is how `Sleep` instances are created. The `Sleep` instances
- /// can either be created directly or the `Handle` instance can be passed to
- /// `with_default`, setting the timer as the default timer for the execution
- /// context.
- pub(crate) fn handle(&self) -> Handle {
- self.handle.clone()
- }
+ fn park_internal(&mut self, rt_handle: &driver::Handle, limit: Option<Duration>) {
+ let handle = rt_handle.time();
+ let mut lock = handle.inner.state.lock();
- fn park_internal(&mut self, limit: Option<Duration>) -> Result<(), P::Error> {
- let mut lock = self.handle.get().state.lock();
-
- assert!(!self.handle.is_shutdown());
+ assert!(!handle.is_shutdown());
let next_wake = lock.wheel.next_expiration_time();
lock.next_wake =
@@ -215,67 +186,62 @@
match next_wake {
Some(when) => {
- let now = self.time_source.now();
+ let now = handle.time_source.now();
// Note that we effectively round up to 1ms here - this avoids
// very short-duration microsecond-resolution sleeps that the OS
// might treat as zero-length.
- let mut duration = self.time_source.tick_to_duration(when.saturating_sub(now));
+ let mut duration = handle
+ .time_source
+ .tick_to_duration(when.saturating_sub(now));
if duration > Duration::from_millis(0) {
if let Some(limit) = limit {
duration = std::cmp::min(limit, duration);
}
- self.park_timeout(duration)?;
+ self.park_thread_timeout(rt_handle, duration);
} else {
- self.park.park_timeout(Duration::from_secs(0))?;
+ self.park.park_timeout(rt_handle, Duration::from_secs(0));
}
}
None => {
if let Some(duration) = limit {
- self.park_timeout(duration)?;
+ self.park_thread_timeout(rt_handle, duration);
} else {
- self.park.park()?;
+ self.park.park(rt_handle);
}
}
}
// Process pending timers after waking up
- self.handle.process();
-
- Ok(())
+ handle.process();
}
cfg_test_util! {
- fn park_timeout(&mut self, duration: Duration) -> Result<(), P::Error> {
- let clock = &self.time_source.clock;
+ fn park_thread_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) {
+ let handle = rt_handle.time();
+ let clock = &handle.time_source.clock;
- if clock.is_paused() {
- self.park.park_timeout(Duration::from_secs(0))?;
+ if clock.can_auto_advance() {
+ self.park.park_timeout(rt_handle, Duration::from_secs(0));
// If the time driver was woken, then the park completed
// before the "duration" elapsed (usually caused by a
// yield in `Runtime::block_on`). In this case, we don't
// advance the clock.
- if !self.did_wake() {
+ if !handle.did_wake() {
// Simulate advancing time
clock.advance(duration);
}
} else {
- self.park.park_timeout(duration)?;
+ self.park.park_timeout(rt_handle, duration);
}
-
- Ok(())
- }
-
- fn did_wake(&self) -> bool {
- self.did_wake.swap(false, Ordering::SeqCst)
}
}
cfg_not_test_util! {
- fn park_timeout(&mut self, duration: Duration) -> Result<(), P::Error> {
- self.park.park_timeout(duration)
+ fn park_thread_timeout(&mut self, rt_handle: &driver::Handle, duration: Duration) {
+ self.park.park_timeout(rt_handle, duration);
}
}
}
@@ -292,7 +258,7 @@
let mut waker_list: [Option<Waker>; 32] = Default::default();
let mut waker_idx = 0;
- let mut lock = self.get().lock();
+ let mut lock = self.inner.lock();
if now < lock.elapsed {
// Time went backwards! This normally shouldn't happen as the Rust language
@@ -323,7 +289,7 @@
waker_idx = 0;
- lock = self.get().lock();
+ lock = self.inner.lock();
}
}
}
@@ -354,7 +320,7 @@
/// `add_entry` must not be called concurrently.
pub(self) unsafe fn clear_entry(&self, entry: NonNull<TimerShared>) {
unsafe {
- let mut lock = self.get().lock();
+ let mut lock = self.inner.lock();
if entry.as_ref().might_be_registered() {
lock.wheel.remove(entry);
@@ -370,9 +336,14 @@
/// driver. No other threads are allowed to concurrently manipulate the
/// timer at all (the current thread should hold an exclusive reference to
/// the `TimerEntry`)
- pub(self) unsafe fn reregister(&self, new_tick: u64, entry: NonNull<TimerShared>) {
+ pub(self) unsafe fn reregister(
+ &self,
+ unpark: &IoHandle,
+ new_tick: u64,
+ entry: NonNull<TimerShared>,
+ ) {
let waker = unsafe {
- let mut lock = self.get().lock();
+ let mut lock = self.inner.lock();
// We may have raced with a firing/deregistration, so check before
// deregistering.
@@ -398,12 +369,12 @@
.map(|next_wake| when < next_wake.get())
.unwrap_or(true)
{
- lock.unpark.unpark();
+ unpark.unpark();
}
None
}
- Err((entry, super::error::InsertError::Elapsed)) => unsafe {
+ Err((entry, crate::time::error::InsertError::Elapsed)) => unsafe {
entry.fire(Ok(()))
},
}
@@ -419,94 +390,17 @@
waker.wake();
}
}
-}
-impl<P> Park for Driver<P>
-where
- P: Park + 'static,
-{
- type Unpark = TimerUnpark<P>;
- type Error = P::Error;
-
- fn unpark(&self) -> Self::Unpark {
- TimerUnpark::new(self)
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- self.park_internal(None)
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.park_internal(Some(duration))
- }
-
- fn shutdown(&mut self) {
- if self.handle.is_shutdown() {
- return;
+ cfg_test_util! {
+ fn did_wake(&self) -> bool {
+ self.inner.did_wake.swap(false, Ordering::SeqCst)
}
-
- self.handle.get().is_shutdown.store(true, Ordering::SeqCst);
-
- // Advance time forward to the end of time.
-
- self.handle.process_at_time(u64::MAX);
-
- self.park.shutdown();
- }
-}
-
-impl<P> Drop for Driver<P>
-where
- P: Park + 'static,
-{
- fn drop(&mut self) {
- self.shutdown();
- }
-}
-
-pub(crate) struct TimerUnpark<P: Park + 'static> {
- inner: P::Unpark,
-
- #[cfg(feature = "test-util")]
- did_wake: Arc<AtomicBool>,
-}
-
-impl<P: Park + 'static> TimerUnpark<P> {
- fn new(driver: &Driver<P>) -> TimerUnpark<P> {
- TimerUnpark {
- inner: driver.park.unpark(),
-
- #[cfg(feature = "test-util")]
- did_wake: driver.did_wake.clone(),
- }
- }
-}
-
-impl<P: Park + 'static> Unpark for TimerUnpark<P> {
- fn unpark(&self) {
- #[cfg(feature = "test-util")]
- self.did_wake.store(true, Ordering::SeqCst);
-
- self.inner.unpark();
}
}
// ===== impl Inner =====
impl Inner {
- pub(self) fn new(time_source: ClockTime, unpark: Box<dyn Unpark>) -> Self {
- Inner {
- state: Mutex::new(InnerState {
- time_source,
- elapsed: 0,
- next_wake: None,
- unpark,
- wheel: wheel::Wheel::new(),
- }),
- is_shutdown: AtomicBool::new(false),
- }
- }
-
/// Locks the driver's inner structure
pub(super) fn lock(&self) -> crate::loom::sync::MutexGuard<'_, InnerState> {
self.state.lock()
diff --git a/src/runtime/time/source.rs b/src/runtime/time/source.rs
new file mode 100644
index 0000000..e6788ed
--- /dev/null
+++ b/src/runtime/time/source.rs
@@ -0,0 +1,42 @@
+use crate::time::{Clock, Duration, Instant};
+
+use std::convert::TryInto;
+
+/// A structure which handles conversion from Instants to u64 timestamps.
+#[derive(Debug)]
+pub(crate) struct TimeSource {
+ pub(crate) clock: Clock,
+ start_time: Instant,
+}
+
+impl TimeSource {
+ pub(crate) fn new(clock: Clock) -> Self {
+ Self {
+ start_time: clock.now(),
+ clock,
+ }
+ }
+
+ pub(crate) fn deadline_to_tick(&self, t: Instant) -> u64 {
+ // Round up to the end of a ms
+ self.instant_to_tick(t + Duration::from_nanos(999_999))
+ }
+
+ pub(crate) fn instant_to_tick(&self, t: Instant) -> u64 {
+ // round up
+ let dur: Duration = t
+ .checked_duration_since(self.start_time)
+ .unwrap_or_else(|| Duration::from_secs(0));
+ let ms = dur.as_millis();
+
+ ms.try_into().unwrap_or(u64::MAX)
+ }
+
+ pub(crate) fn tick_to_duration(&self, t: u64) -> Duration {
+ Duration::from_millis(t)
+ }
+
+ pub(crate) fn now(&self) -> u64 {
+ self.instant_to_tick(self.clock.now())
+ }
+}
diff --git a/src/runtime/time/tests/mod.rs b/src/runtime/time/tests/mod.rs
new file mode 100644
index 0000000..88c7d76
--- /dev/null
+++ b/src/runtime/time/tests/mod.rs
@@ -0,0 +1,264 @@
+#![cfg(not(tokio_wasi))]
+
+use std::{task::Context, time::Duration};
+
+#[cfg(not(loom))]
+use futures::task::noop_waker_ref;
+
+use crate::loom::sync::atomic::{AtomicBool, Ordering};
+use crate::loom::sync::Arc;
+use crate::loom::thread;
+
+use super::TimerEntry;
+
+fn block_on<T>(f: impl std::future::Future<Output = T>) -> T {
+ #[cfg(loom)]
+ return loom::future::block_on(f);
+
+ #[cfg(not(loom))]
+ {
+ let rt = crate::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap();
+ rt.block_on(f)
+ }
+}
+
+fn model(f: impl Fn() + Send + Sync + 'static) {
+ #[cfg(loom)]
+ loom::model(f);
+
+ #[cfg(not(loom))]
+ f();
+}
+
+fn rt(start_paused: bool) -> crate::runtime::Runtime {
+ crate::runtime::Builder::new_current_thread()
+ .enable_time()
+ .start_paused(start_paused)
+ .build()
+ .unwrap()
+}
+
+#[test]
+fn single_timer() {
+ model(|| {
+ let rt = rt(false);
+ let handle = rt.handle();
+
+ let handle_ = handle.clone();
+ let jh = thread::spawn(move || {
+ let entry = TimerEntry::new(
+ &handle_.inner,
+ handle_.inner.driver().clock().now() + Duration::from_secs(1),
+ );
+ pin!(entry);
+
+ block_on(futures::future::poll_fn(|cx| {
+ entry.as_mut().poll_elapsed(cx)
+ }))
+ .unwrap();
+ });
+
+ thread::yield_now();
+
+ let handle = handle.inner.driver().time();
+
+ // This may or may not return Some (depending on how it races with the
+ // thread). If it does return None, however, the timer should complete
+ // synchronously.
+ handle.process_at_time(handle.time_source().now() + 2_000_000_000);
+
+ jh.join().unwrap();
+ })
+}
+
+#[test]
+fn drop_timer() {
+ model(|| {
+ let rt = rt(false);
+ let handle = rt.handle();
+
+ let handle_ = handle.clone();
+ let jh = thread::spawn(move || {
+ let entry = TimerEntry::new(
+ &handle_.inner,
+ handle_.inner.driver().clock().now() + Duration::from_secs(1),
+ );
+ pin!(entry);
+
+ let _ = entry
+ .as_mut()
+ .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
+ let _ = entry
+ .as_mut()
+ .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
+ });
+
+ thread::yield_now();
+
+ let handle = handle.inner.driver().time();
+
+ // advance 2s in the future.
+ handle.process_at_time(handle.time_source().now() + 2_000_000_000);
+
+ jh.join().unwrap();
+ })
+}
+
+#[test]
+fn change_waker() {
+ model(|| {
+ let rt = rt(false);
+ let handle = rt.handle();
+
+ let handle_ = handle.clone();
+ let jh = thread::spawn(move || {
+ let entry = TimerEntry::new(
+ &handle_.inner,
+ handle_.inner.driver().clock().now() + Duration::from_secs(1),
+ );
+ pin!(entry);
+
+ let _ = entry
+ .as_mut()
+ .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
+
+ block_on(futures::future::poll_fn(|cx| {
+ entry.as_mut().poll_elapsed(cx)
+ }))
+ .unwrap();
+ });
+
+ thread::yield_now();
+
+ let handle = handle.inner.driver().time();
+
+ // advance 2s
+ handle.process_at_time(handle.time_source().now() + 2_000_000_000);
+
+ jh.join().unwrap();
+ })
+}
+
+#[test]
+fn reset_future() {
+ model(|| {
+ let finished_early = Arc::new(AtomicBool::new(false));
+
+ let rt = rt(false);
+ let handle = rt.handle();
+
+ let handle_ = handle.clone();
+ let finished_early_ = finished_early.clone();
+ let start = handle.inner.driver().clock().now();
+
+ let jh = thread::spawn(move || {
+ let entry = TimerEntry::new(&handle_.inner, start + Duration::from_secs(1));
+ pin!(entry);
+
+ let _ = entry
+ .as_mut()
+ .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
+
+ entry.as_mut().reset(start + Duration::from_secs(2));
+
+ // shouldn't complete before 2s
+ block_on(futures::future::poll_fn(|cx| {
+ entry.as_mut().poll_elapsed(cx)
+ }))
+ .unwrap();
+
+ finished_early_.store(true, Ordering::Relaxed);
+ });
+
+ thread::yield_now();
+
+ let handle = handle.inner.driver().time();
+
+ // This may or may not return a wakeup time.
+ handle.process_at_time(
+ handle
+ .time_source()
+ .instant_to_tick(start + Duration::from_millis(1500)),
+ );
+
+ assert!(!finished_early.load(Ordering::Relaxed));
+
+ handle.process_at_time(
+ handle
+ .time_source()
+ .instant_to_tick(start + Duration::from_millis(2500)),
+ );
+
+ jh.join().unwrap();
+
+ assert!(finished_early.load(Ordering::Relaxed));
+ })
+}
+
+#[cfg(not(loom))]
+fn normal_or_miri<T>(normal: T, miri: T) -> T {
+ if cfg!(miri) {
+ miri
+ } else {
+ normal
+ }
+}
+
+#[test]
+#[cfg(not(loom))]
+fn poll_process_levels() {
+ let rt = rt(true);
+ let handle = rt.handle();
+
+ let mut entries = vec![];
+
+ for i in 0..normal_or_miri(1024, 64) {
+ let mut entry = Box::pin(TimerEntry::new(
+ &handle.inner,
+ handle.inner.driver().clock().now() + Duration::from_millis(i),
+ ));
+
+ let _ = entry
+ .as_mut()
+ .poll_elapsed(&mut Context::from_waker(noop_waker_ref()));
+
+ entries.push(entry);
+ }
+
+ for t in 1..normal_or_miri(1024, 64) {
+ handle.inner.driver().time().process_at_time(t as u64);
+
+ for (deadline, future) in entries.iter_mut().enumerate() {
+ let mut context = Context::from_waker(noop_waker_ref());
+ if deadline <= t {
+ assert!(future.as_mut().poll_elapsed(&mut context).is_ready());
+ } else {
+ assert!(future.as_mut().poll_elapsed(&mut context).is_pending());
+ }
+ }
+ }
+}
+
+#[test]
+#[cfg(not(loom))]
+fn poll_process_levels_targeted() {
+ let mut context = Context::from_waker(noop_waker_ref());
+
+ let rt = rt(true);
+ let handle = rt.handle();
+
+ let e1 = TimerEntry::new(
+ &handle.inner,
+ handle.inner.driver().clock().now() + Duration::from_millis(193),
+ );
+ pin!(e1);
+
+ let handle = handle.inner.driver().time();
+
+ handle.process_at_time(62);
+ assert!(e1.as_mut().poll_elapsed(&mut context).is_pending());
+ handle.process_at_time(192);
+ handle.process_at_time(192);
+}
diff --git a/src/time/driver/wheel/level.rs b/src/runtime/time/wheel/level.rs
similarity index 96%
rename from src/time/driver/wheel/level.rs
rename to src/runtime/time/wheel/level.rs
index 34d3176..7e48ff5 100644
--- a/src/time/driver/wheel/level.rs
+++ b/src/runtime/time/wheel/level.rs
@@ -1,6 +1,4 @@
-use crate::time::driver::TimerHandle;
-
-use crate::time::driver::{EntryList, TimerShared};
+use crate::runtime::time::{EntryList, TimerHandle, TimerShared};
use std::{fmt, ptr::NonNull};
@@ -53,7 +51,7 @@
// However, that is only supported for arrays of size
// 32 or fewer. So in our case we have to explicitly
// invoke the constructor for each array element.
- let ctor = || EntryList::default();
+ let ctor = EntryList::default;
Level {
level,
@@ -143,8 +141,9 @@
let level_range = level_range(self.level);
let slot_range = slot_range(self.level);
- // TODO: This can probably be simplified w/ power of 2 math
- let level_start = now - (now % level_range);
+ // Compute the start date of the current level by masking the low bits
+ // of `now` (`level_range` is a power of 2).
+ let level_start = now & !(level_range - 1);
let mut deadline = level_start + slot as u64 * slot_range;
if deadline <= now {
diff --git a/src/time/driver/wheel/mod.rs b/src/runtime/time/wheel/mod.rs
similarity index 99%
rename from src/time/driver/wheel/mod.rs
rename to src/runtime/time/wheel/mod.rs
index f088f2c..c3ba364 100644
--- a/src/time/driver/wheel/mod.rs
+++ b/src/runtime/time/wheel/mod.rs
@@ -1,4 +1,4 @@
-use crate::time::driver::{TimerHandle, TimerShared};
+use crate::runtime::time::{TimerHandle, TimerShared};
use crate::time::error::InsertError;
mod level;
diff --git a/src/signal/mod.rs b/src/signal/mod.rs
index 882218a..3aacc60 100644
--- a/src/signal/mod.rs
+++ b/src/signal/mod.rs
@@ -48,7 +48,7 @@
mod ctrl_c;
pub use ctrl_c::ctrl_c;
-mod registry;
+pub(crate) mod registry;
mod os {
#[cfg(unix)]
diff --git a/src/signal/registry.rs b/src/signal/registry.rs
index e0a2df9..48e98c8 100644
--- a/src/signal/registry.rs
+++ b/src/signal/registry.rs
@@ -1,12 +1,10 @@
#![allow(clippy::unit_arg)]
use crate::signal::os::{OsExtraData, OsStorage};
-
use crate::sync::watch;
+use crate::util::once_cell::OnceCell;
-use once_cell::sync::Lazy;
use std::ops;
-use std::pin::Pin;
use std::sync::atomic::{AtomicBool, Ordering};
pub(crate) type EventId = usize;
@@ -152,19 +150,25 @@
}
}
-pub(crate) fn globals() -> Pin<&'static Globals>
+fn globals_init() -> Globals
where
OsExtraData: 'static + Send + Sync + Init,
OsStorage: 'static + Send + Sync + Init,
{
- static GLOBALS: Lazy<Pin<Box<Globals>>> = Lazy::new(|| {
- Box::pin(Globals {
- extra: OsExtraData::init(),
- registry: Registry::new(OsStorage::init()),
- })
- });
+ Globals {
+ extra: OsExtraData::init(),
+ registry: Registry::new(OsStorage::init()),
+ }
+}
- GLOBALS.as_ref()
+pub(crate) fn globals() -> &'static Globals
+where
+ OsExtraData: 'static + Send + Sync + Init,
+ OsStorage: 'static + Send + Sync + Init,
+{
+ static GLOBALS: OnceCell<Globals> = OnceCell::new();
+
+ GLOBALS.get(globals_init)
}
#[cfg(all(test, not(loom)))]
@@ -202,7 +206,12 @@
registry.broadcast();
// Yield so the previous broadcast can get received
- crate::time::sleep(std::time::Duration::from_millis(10)).await;
+ //
+ // This yields many times since the block_on task is only polled every 61
+ // ticks.
+ for _ in 0..100 {
+ crate::task::yield_now().await;
+ }
// Send subsequent signal
registry.record_event(0);
@@ -232,7 +241,7 @@
#[test]
fn record_invalid_event_does_nothing() {
let registry = Registry::new(vec![EventInfo::default()]);
- registry.record_event(42);
+ registry.record_event(1302);
}
#[test]
diff --git a/src/signal/unix.rs b/src/signal/unix.rs
index 86ea9a9..e5345fd 100644
--- a/src/signal/unix.rs
+++ b/src/signal/unix.rs
@@ -6,29 +6,31 @@
#![cfg(unix)]
#![cfg_attr(docsrs, doc(cfg(all(unix, feature = "signal"))))]
+use crate::runtime::scheduler;
+use crate::runtime::signal::Handle;
use crate::signal::registry::{globals, EventId, EventInfo, Globals, Init, Storage};
use crate::signal::RxFuture;
use crate::sync::watch;
use mio::net::UnixStream;
use std::io::{self, Error, ErrorKind, Write};
-use std::pin::Pin;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Once;
use std::task::{Context, Poll};
-pub(crate) mod driver;
-use self::driver::Handle;
-
pub(crate) type OsStorage = Vec<SignalInfo>;
-// Number of different unix signals
-// (FreeBSD has 33)
-const SIGNUM: usize = 33;
-
impl Init for OsStorage {
fn init() -> Self {
- (0..SIGNUM).map(|_| SignalInfo::default()).collect()
+ // There are reliable signals ranging from 1 to 33 available on every Unix platform.
+ #[cfg(not(target_os = "linux"))]
+ let possible = 0..=33;
+
+ // On Linux, there are additional real-time signals available.
+ #[cfg(target_os = "linux")]
+ let possible = 0..=libc::SIGRTMAX();
+
+ possible.map(|_| SignalInfo::default()).collect()
}
}
@@ -48,7 +50,7 @@
#[derive(Debug)]
pub(crate) struct OsExtraData {
sender: UnixStream,
- receiver: UnixStream,
+ pub(crate) receiver: UnixStream,
}
impl Init for OsExtraData {
@@ -60,7 +62,7 @@
}
/// Represents the specific kind of signal to listen for.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
pub struct SignalKind(libc::c_int);
impl SignalKind {
@@ -80,15 +82,26 @@
// unlikely to change to other types, but technically libc can change this
// in the future minor version.
// See https://github.com/tokio-rs/tokio/issues/3767 for more.
- pub fn from_raw(signum: std::os::raw::c_int) -> Self {
+ pub const fn from_raw(signum: std::os::raw::c_int) -> Self {
Self(signum as libc::c_int)
}
+ /// Get the signal's numeric value.
+ ///
+ /// ```rust
+ /// # use tokio::signal::unix::SignalKind;
+ /// let kind = SignalKind::interrupt();
+ /// assert_eq!(kind.as_raw_value(), libc::SIGINT);
+ /// ```
+ pub const fn as_raw_value(&self) -> std::os::raw::c_int {
+ self.0
+ }
+
/// Represents the SIGALRM signal.
///
/// On Unix systems this signal is sent when a real-time timer has expired.
/// By default, the process is terminated by this signal.
- pub fn alarm() -> Self {
+ pub const fn alarm() -> Self {
Self(libc::SIGALRM)
}
@@ -96,7 +109,7 @@
///
/// On Unix systems this signal is sent when the status of a child process
/// has changed. By default, this signal is ignored.
- pub fn child() -> Self {
+ pub const fn child() -> Self {
Self(libc::SIGCHLD)
}
@@ -104,7 +117,7 @@
///
/// On Unix systems this signal is sent when the terminal is disconnected.
/// By default, the process is terminated by this signal.
- pub fn hangup() -> Self {
+ pub const fn hangup() -> Self {
Self(libc::SIGHUP)
}
@@ -119,7 +132,7 @@
target_os = "netbsd",
target_os = "openbsd"
))]
- pub fn info() -> Self {
+ pub const fn info() -> Self {
Self(libc::SIGINFO)
}
@@ -127,7 +140,7 @@
///
/// On Unix systems this signal is sent to interrupt a program.
/// By default, the process is terminated by this signal.
- pub fn interrupt() -> Self {
+ pub const fn interrupt() -> Self {
Self(libc::SIGINT)
}
@@ -135,7 +148,7 @@
///
/// On Unix systems this signal is sent when I/O operations are possible
/// on some file descriptor. By default, this signal is ignored.
- pub fn io() -> Self {
+ pub const fn io() -> Self {
Self(libc::SIGIO)
}
@@ -144,7 +157,7 @@
/// On Unix systems this signal is sent when the process attempts to write
/// to a pipe which has no reader. By default, the process is terminated by
/// this signal.
- pub fn pipe() -> Self {
+ pub const fn pipe() -> Self {
Self(libc::SIGPIPE)
}
@@ -153,7 +166,7 @@
/// On Unix systems this signal is sent to issue a shutdown of the
/// process, after which the OS will dump the process core.
/// By default, the process is terminated by this signal.
- pub fn quit() -> Self {
+ pub const fn quit() -> Self {
Self(libc::SIGQUIT)
}
@@ -161,7 +174,7 @@
///
/// On Unix systems this signal is sent to issue a shutdown of the
/// process. By default, the process is terminated by this signal.
- pub fn terminate() -> Self {
+ pub const fn terminate() -> Self {
Self(libc::SIGTERM)
}
@@ -169,7 +182,7 @@
///
/// On Unix systems this is a user defined signal.
/// By default, the process is terminated by this signal.
- pub fn user_defined1() -> Self {
+ pub const fn user_defined1() -> Self {
Self(libc::SIGUSR1)
}
@@ -177,7 +190,7 @@
///
/// On Unix systems this is a user defined signal.
/// By default, the process is terminated by this signal.
- pub fn user_defined2() -> Self {
+ pub const fn user_defined2() -> Self {
Self(libc::SIGUSR2)
}
@@ -185,11 +198,23 @@
///
/// On Unix systems this signal is sent when the terminal window is resized.
/// By default, this signal is ignored.
- pub fn window_change() -> Self {
+ pub const fn window_change() -> Self {
Self(libc::SIGWINCH)
}
}
+impl From<std::os::raw::c_int> for SignalKind {
+ fn from(signum: std::os::raw::c_int) -> Self {
+ Self::from_raw(signum as libc::c_int)
+ }
+}
+
+impl From<SignalKind> for std::os::raw::c_int {
+ fn from(kind: SignalKind) -> Self {
+ kind.as_raw_value()
+ }
+}
+
pub(crate) struct SignalInfo {
event_info: EventInfo,
init: Once,
@@ -214,7 +239,7 @@
/// 2. Wake up the driver by writing a byte to a pipe
///
/// Those two operations should both be async-signal safe.
-fn action(globals: Pin<&'static Globals>, signal: libc::c_int) {
+fn action(globals: &'static Globals, signal: libc::c_int) {
globals.record_event(signal as EventId);
// Send a wakeup, ignore any errors (anything reasonably possible is
@@ -357,8 +382,15 @@
/// * If the previous initialization of this specific signal failed.
/// * If the signal is one of
/// [`signal_hook::FORBIDDEN`](fn@signal_hook_registry::register#panics)
+///
+/// # Panics
+///
+/// This function panics if there is no current reactor set, or if the `rt`
+/// feature flag is not enabled.
+#[track_caller]
pub fn signal(kind: SignalKind) -> io::Result<Signal> {
- let rx = signal_with_handle(kind, &Handle::current())?;
+ let handle = scheduler::Handle::current();
+ let rx = signal_with_handle(kind, handle.driver().signal())?;
Ok(Signal {
inner: RxFuture::new(rx),
@@ -380,6 +412,12 @@
///
/// `None` is returned if no more events can be received by this stream.
///
+ /// # Cancel safety
+ ///
+ /// This method is cancel safe. If you use it as the event in a
+ /// [`tokio::select!`](crate::select) statement and some other branch
+ /// completes first, then it is guaranteed that no signal is lost.
+ ///
/// # Examples
///
/// Wait for SIGHUP
@@ -474,4 +512,15 @@
)
.unwrap_err();
}
+
+ #[test]
+ fn from_c_int() {
+ assert_eq!(SignalKind::from(2), SignalKind::interrupt());
+ }
+
+ #[test]
+ fn into_c_int() {
+ let value: std::os::raw::c_int = SignalKind::interrupt().into();
+ assert_eq!(value, libc::SIGINT as _);
+ }
}
diff --git a/src/signal/unix/driver.rs b/src/signal/unix/driver.rs
deleted file mode 100644
index 5fe7c35..0000000
--- a/src/signal/unix/driver.rs
+++ /dev/null
@@ -1,207 +0,0 @@
-#![cfg_attr(not(feature = "rt"), allow(dead_code))]
-
-//! Signal driver
-
-use crate::io::driver::{Driver as IoDriver, Interest};
-use crate::io::PollEvented;
-use crate::park::Park;
-use crate::signal::registry::globals;
-
-use mio::net::UnixStream;
-use std::io::{self, Read};
-use std::ptr;
-use std::sync::{Arc, Weak};
-use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
-use std::time::Duration;
-
-/// Responsible for registering wakeups when an OS signal is received, and
-/// subsequently dispatching notifications to any signal listeners as appropriate.
-///
-/// Note: this driver relies on having an enabled IO driver in order to listen to
-/// pipe write wakeups.
-#[derive(Debug)]
-pub(crate) struct Driver {
- /// Thread parker. The `Driver` park implementation delegates to this.
- park: IoDriver,
-
- /// A pipe for receiving wake events from the signal handler
- receiver: PollEvented<UnixStream>,
-
- /// Shared state
- inner: Arc<Inner>,
-}
-
-#[derive(Clone, Debug, Default)]
-pub(crate) struct Handle {
- inner: Weak<Inner>,
-}
-
-#[derive(Debug)]
-pub(super) struct Inner(());
-
-// ===== impl Driver =====
-
-impl Driver {
- /// Creates a new signal `Driver` instance that delegates wakeups to `park`.
- pub(crate) fn new(park: IoDriver) -> io::Result<Self> {
- use std::mem::ManuallyDrop;
- use std::os::unix::io::{AsRawFd, FromRawFd};
-
- // NB: We give each driver a "fresh" receiver file descriptor to avoid
- // the issues described in alexcrichton/tokio-process#42.
- //
- // In the past we would reuse the actual receiver file descriptor and
- // swallow any errors around double registration of the same descriptor.
- // I'm not sure if the second (failed) registration simply doesn't end
- // up receiving wake up notifications, or there could be some race
- // condition when consuming readiness events, but having distinct
- // descriptors for distinct PollEvented instances appears to mitigate
- // this.
- //
- // Unfortunately we cannot just use a single global PollEvented instance
- // either, since we can't compare Handles or assume they will always
- // point to the exact same reactor.
- //
- // Mio 0.7 removed `try_clone()` as an API due to unexpected behavior
- // with registering dups with the same reactor. In this case, duping is
- // safe as each dup is registered with separate reactors **and** we
- // only expect at least one dup to receive the notification.
-
- // Manually drop as we don't actually own this instance of UnixStream.
- let receiver_fd = globals().receiver.as_raw_fd();
-
- // safety: there is nothing unsafe about this, but the `from_raw_fd` fn is marked as unsafe.
- let original =
- ManuallyDrop::new(unsafe { std::os::unix::net::UnixStream::from_raw_fd(receiver_fd) });
- let receiver = UnixStream::from_std(original.try_clone()?);
- let receiver = PollEvented::new_with_interest_and_handle(
- receiver,
- Interest::READABLE | Interest::WRITABLE,
- park.handle(),
- )?;
-
- Ok(Self {
- park,
- receiver,
- inner: Arc::new(Inner(())),
- })
- }
-
- /// Returns a handle to this event loop which can be sent across threads
- /// and can be used as a proxy to the event loop itself.
- pub(crate) fn handle(&self) -> Handle {
- Handle {
- inner: Arc::downgrade(&self.inner),
- }
- }
-
- fn process(&self) {
- // Check if the pipe is ready to read and therefore has "woken" us up
- //
- // To do so, we will `poll_read_ready` with a noop waker, since we don't
- // need to actually be notified when read ready...
- let waker = unsafe { Waker::from_raw(RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE)) };
- let mut cx = Context::from_waker(&waker);
-
- let ev = match self.receiver.registration().poll_read_ready(&mut cx) {
- Poll::Ready(Ok(ev)) => ev,
- Poll::Ready(Err(e)) => panic!("reactor gone: {}", e),
- Poll::Pending => return, // No wake has arrived, bail
- };
-
- // Drain the pipe completely so we can receive a new readiness event
- // if another signal has come in.
- let mut buf = [0; 128];
- loop {
- match (&*self.receiver).read(&mut buf) {
- Ok(0) => panic!("EOF on self-pipe"),
- Ok(_) => continue, // Keep reading
- Err(e) if e.kind() == io::ErrorKind::WouldBlock => break,
- Err(e) => panic!("Bad read on self-pipe: {}", e),
- }
- }
-
- self.receiver.registration().clear_readiness(ev);
-
- // Broadcast any signals which were received
- globals().broadcast();
- }
-}
-
-const NOOP_WAKER_VTABLE: RawWakerVTable = RawWakerVTable::new(noop_clone, noop, noop, noop);
-
-unsafe fn noop_clone(_data: *const ()) -> RawWaker {
- RawWaker::new(ptr::null(), &NOOP_WAKER_VTABLE)
-}
-
-unsafe fn noop(_data: *const ()) {}
-
-// ===== impl Park for Driver =====
-
-impl Park for Driver {
- type Unpark = <IoDriver as Park>::Unpark;
- type Error = io::Error;
-
- fn unpark(&self) -> Self::Unpark {
- self.park.unpark()
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- self.park.park()?;
- self.process();
- Ok(())
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.park.park_timeout(duration)?;
- self.process();
- Ok(())
- }
-
- fn shutdown(&mut self) {
- self.park.shutdown()
- }
-}
-
-// ===== impl Handle =====
-
-impl Handle {
- pub(super) fn check_inner(&self) -> io::Result<()> {
- if self.inner.strong_count() > 0 {
- Ok(())
- } else {
- Err(io::Error::new(io::ErrorKind::Other, "signal driver gone"))
- }
- }
-}
-
-cfg_rt! {
- impl Handle {
- /// Returns a handle to the current driver
- ///
- /// # Panics
- ///
- /// This function panics if there is no current signal driver set.
- pub(super) fn current() -> Self {
- crate::runtime::context::signal_handle().expect(
- "there is no signal driver running, must be called from the context of Tokio runtime",
- )
- }
- }
-}
-
-cfg_not_rt! {
- impl Handle {
- /// Returns a handle to the current driver
- ///
- /// # Panics
- ///
- /// This function panics if there is no current signal driver set.
- pub(super) fn current() -> Self {
- panic!(
- "there is no signal driver running, must be called from the context of Tokio runtime or with\
- `rt` enabled.",
- )
- }
- }
-}
diff --git a/src/signal/windows.rs b/src/signal/windows.rs
index 11ec6cb..730f95d 100644
--- a/src/signal/windows.rs
+++ b/src/signal/windows.rs
@@ -1,9 +1,9 @@
//! Windows-specific types for signal handling.
//!
-//! This module is only defined on Windows and allows receiving "ctrl-c"
-//! and "ctrl-break" notifications. These events are listened for via the
-//! `SetConsoleCtrlHandler` function which receives events of the type
-//! `CTRL_C_EVENT` and `CTRL_BREAK_EVENT`.
+//! This module is only defined on Windows and allows receiving "ctrl-c",
+//! "ctrl-break", "ctrl-logoff", "ctrl-shutdown", and "ctrl-close"
+//! notifications. These events are listened for via the `SetConsoleCtrlHandler`
+//! function which receives the corresponding windows_sys event type.
#![cfg(any(windows, docsrs))]
#![cfg_attr(docsrs, doc(cfg(all(windows, feature = "signal"))))]
@@ -221,3 +221,297 @@
inner: self::imp::ctrl_break()?,
})
}
+
+/// Creates a new stream which receives "ctrl-close" notifications sent to the
+/// process.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use tokio::signal::windows::ctrl_close;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// // An infinite stream of CTRL-CLOSE events.
+/// let mut stream = ctrl_close()?;
+///
+/// // Print whenever a CTRL-CLOSE event is received.
+/// for countdown in (0..3).rev() {
+/// stream.recv().await;
+/// println!("got CTRL-CLOSE. {} more to exit", countdown);
+/// }
+///
+/// Ok(())
+/// }
+/// ```
+pub fn ctrl_close() -> io::Result<CtrlClose> {
+ Ok(CtrlClose {
+ inner: self::imp::ctrl_close()?,
+ })
+}
+
+/// Represents a stream which receives "ctrl-close" notitifications sent to the process
+/// via 'SetConsoleCtrlHandler'.
+///
+/// A notification to this process notifies *all* streams listening for
+/// this event. Moreover, the notifications **are coalesced** if they aren't processed
+/// quickly enough. This means that if two notifications are received back-to-back,
+/// then the stream may only receive one item about the two notifications.
+#[must_use = "streams do nothing unless polled"]
+#[derive(Debug)]
+pub struct CtrlClose {
+ inner: RxFuture,
+}
+
+impl CtrlClose {
+ /// Receives the next signal notification event.
+ ///
+ /// `None` is returned if no more events can be received by this stream.
+ ///
+ /// # Examples
+ ///
+ /// ```rust,no_run
+ /// use tokio::signal::windows::ctrl_close;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// // An infinite stream of CTRL-CLOSE events.
+ /// let mut stream = ctrl_close()?;
+ ///
+ /// // Print whenever a CTRL-CLOSE event is received.
+ /// stream.recv().await;
+ /// println!("got CTRL-CLOSE. Cleaning up before exiting");
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn recv(&mut self) -> Option<()> {
+ self.inner.recv().await
+ }
+
+ /// Polls to receive the next signal notification event, outside of an
+ /// `async` context.
+ ///
+ /// `None` is returned if no more events can be received by this stream.
+ ///
+ /// # Examples
+ ///
+ /// Polling from a manually implemented future
+ ///
+ /// ```rust,no_run
+ /// use std::pin::Pin;
+ /// use std::future::Future;
+ /// use std::task::{Context, Poll};
+ /// use tokio::signal::windows::CtrlClose;
+ ///
+ /// struct MyFuture {
+ /// ctrl_close: CtrlClose,
+ /// }
+ ///
+ /// impl Future for MyFuture {
+ /// type Output = Option<()>;
+ ///
+ /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ /// println!("polling MyFuture");
+ /// self.ctrl_close.poll_recv(cx)
+ /// }
+ /// }
+ /// ```
+ pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<()>> {
+ self.inner.poll_recv(cx)
+ }
+}
+
+/// Creates a new stream which receives "ctrl-shutdown" notifications sent to the
+/// process.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use tokio::signal::windows::ctrl_shutdown;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// // An infinite stream of CTRL-SHUTDOWN events.
+/// let mut stream = ctrl_shutdown()?;
+///
+/// stream.recv().await;
+/// println!("got CTRL-SHUTDOWN. Cleaning up before exiting");
+///
+/// Ok(())
+/// }
+/// ```
+pub fn ctrl_shutdown() -> io::Result<CtrlShutdown> {
+ Ok(CtrlShutdown {
+ inner: self::imp::ctrl_shutdown()?,
+ })
+}
+
+/// Represents a stream which receives "ctrl-shutdown" notitifications sent to the process
+/// via 'SetConsoleCtrlHandler'.
+///
+/// A notification to this process notifies *all* streams listening for
+/// this event. Moreover, the notifications **are coalesced** if they aren't processed
+/// quickly enough. This means that if two notifications are received back-to-back,
+/// then the stream may only receive one item about the two notifications.
+#[must_use = "streams do nothing unless polled"]
+#[derive(Debug)]
+pub struct CtrlShutdown {
+ inner: RxFuture,
+}
+
+impl CtrlShutdown {
+ /// Receives the next signal notification event.
+ ///
+ /// `None` is returned if no more events can be received by this stream.
+ ///
+ /// # Examples
+ ///
+ /// ```rust,no_run
+ /// use tokio::signal::windows::ctrl_shutdown;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// // An infinite stream of CTRL-SHUTDOWN events.
+ /// let mut stream = ctrl_shutdown()?;
+ ///
+ /// // Print whenever a CTRL-SHUTDOWN event is received.
+ /// stream.recv().await;
+ /// println!("got CTRL-SHUTDOWN. Cleaning up before exiting");
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn recv(&mut self) -> Option<()> {
+ self.inner.recv().await
+ }
+
+ /// Polls to receive the next signal notification event, outside of an
+ /// `async` context.
+ ///
+ /// `None` is returned if no more events can be received by this stream.
+ ///
+ /// # Examples
+ ///
+ /// Polling from a manually implemented future
+ ///
+ /// ```rust,no_run
+ /// use std::pin::Pin;
+ /// use std::future::Future;
+ /// use std::task::{Context, Poll};
+ /// use tokio::signal::windows::CtrlShutdown;
+ ///
+ /// struct MyFuture {
+ /// ctrl_shutdown: CtrlShutdown,
+ /// }
+ ///
+ /// impl Future for MyFuture {
+ /// type Output = Option<()>;
+ ///
+ /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ /// println!("polling MyFuture");
+ /// self.ctrl_shutdown.poll_recv(cx)
+ /// }
+ /// }
+ /// ```
+ pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<()>> {
+ self.inner.poll_recv(cx)
+ }
+}
+
+/// Creates a new stream which receives "ctrl-logoff" notifications sent to the
+/// process.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use tokio::signal::windows::ctrl_logoff;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// // An infinite stream of CTRL-LOGOFF events.
+/// let mut stream = ctrl_logoff()?;
+///
+/// stream.recv().await;
+/// println!("got CTRL-LOGOFF. Cleaning up before exiting");
+///
+/// Ok(())
+/// }
+/// ```
+pub fn ctrl_logoff() -> io::Result<CtrlLogoff> {
+ Ok(CtrlLogoff {
+ inner: self::imp::ctrl_logoff()?,
+ })
+}
+
+/// Represents a stream which receives "ctrl-logoff" notitifications sent to the process
+/// via 'SetConsoleCtrlHandler'.
+///
+/// A notification to this process notifies *all* streams listening for
+/// this event. Moreover, the notifications **are coalesced** if they aren't processed
+/// quickly enough. This means that if two notifications are received back-to-back,
+/// then the stream may only receive one item about the two notifications.
+#[must_use = "streams do nothing unless polled"]
+#[derive(Debug)]
+pub struct CtrlLogoff {
+ inner: RxFuture,
+}
+
+impl CtrlLogoff {
+ /// Receives the next signal notification event.
+ ///
+ /// `None` is returned if no more events can be received by this stream.
+ ///
+ /// # Examples
+ ///
+ /// ```rust,no_run
+ /// use tokio::signal::windows::ctrl_logoff;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// // An infinite stream of CTRL-LOGOFF events.
+ /// let mut stream = ctrl_logoff()?;
+ ///
+ /// // Print whenever a CTRL-LOGOFF event is received.
+ /// stream.recv().await;
+ /// println!("got CTRL-LOGOFF. Cleaning up before exiting");
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn recv(&mut self) -> Option<()> {
+ self.inner.recv().await
+ }
+
+ /// Polls to receive the next signal notification event, outside of an
+ /// `async` context.
+ ///
+ /// `None` is returned if no more events can be received by this stream.
+ ///
+ /// # Examples
+ ///
+ /// Polling from a manually implemented future
+ ///
+ /// ```rust,no_run
+ /// use std::pin::Pin;
+ /// use std::future::Future;
+ /// use std::task::{Context, Poll};
+ /// use tokio::signal::windows::CtrlLogoff;
+ ///
+ /// struct MyFuture {
+ /// ctrl_logoff: CtrlLogoff,
+ /// }
+ ///
+ /// impl Future for MyFuture {
+ /// type Output = Option<()>;
+ ///
+ /// fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ /// println!("polling MyFuture");
+ /// self.ctrl_logoff.poll_recv(cx)
+ /// }
+ /// }
+ /// ```
+ pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<()>> {
+ self.inner.poll_recv(cx)
+ }
+}
diff --git a/src/signal/windows/stub.rs b/src/signal/windows/stub.rs
index 8863054..61df309 100644
--- a/src/signal/windows/stub.rs
+++ b/src/signal/windows/stub.rs
@@ -4,10 +4,22 @@
use crate::signal::RxFuture;
use std::io;
+pub(super) fn ctrl_break() -> io::Result<RxFuture> {
+ panic!()
+}
+
+pub(super) fn ctrl_close() -> io::Result<RxFuture> {
+ panic!()
+}
+
pub(super) fn ctrl_c() -> io::Result<RxFuture> {
panic!()
}
-pub(super) fn ctrl_break() -> io::Result<RxFuture> {
+pub(super) fn ctrl_logoff() -> io::Result<RxFuture> {
+ panic!()
+}
+
+pub(super) fn ctrl_shutdown() -> io::Result<RxFuture> {
panic!()
}
diff --git a/src/signal/windows/sys.rs b/src/signal/windows/sys.rs
index 8d29c35..f8133e0 100644
--- a/src/signal/windows/sys.rs
+++ b/src/signal/windows/sys.rs
@@ -5,19 +5,30 @@
use crate::signal::registry::{globals, EventId, EventInfo, Init, Storage};
use crate::signal::RxFuture;
-use winapi::shared::minwindef::{BOOL, DWORD, FALSE, TRUE};
-use winapi::um::consoleapi::SetConsoleCtrlHandler;
-use winapi::um::wincon::{CTRL_BREAK_EVENT, CTRL_C_EVENT};
-
-pub(super) fn ctrl_c() -> io::Result<RxFuture> {
- new(CTRL_C_EVENT)
-}
+use windows_sys::Win32::Foundation::BOOL;
+use windows_sys::Win32::System::Console as console;
pub(super) fn ctrl_break() -> io::Result<RxFuture> {
- new(CTRL_BREAK_EVENT)
+ new(console::CTRL_BREAK_EVENT)
}
-fn new(signum: DWORD) -> io::Result<RxFuture> {
+pub(super) fn ctrl_close() -> io::Result<RxFuture> {
+ new(console::CTRL_CLOSE_EVENT)
+}
+
+pub(super) fn ctrl_c() -> io::Result<RxFuture> {
+ new(console::CTRL_C_EVENT)
+}
+
+pub(super) fn ctrl_logoff() -> io::Result<RxFuture> {
+ new(console::CTRL_LOGOFF_EVENT)
+}
+
+pub(super) fn ctrl_shutdown() -> io::Result<RxFuture> {
+ new(console::CTRL_SHUTDOWN_EVENT)
+}
+
+fn new(signum: u32) -> io::Result<RxFuture> {
global_init()?;
let rx = globals().register_listener(signum as EventId);
Ok(RxFuture::new(rx))
@@ -25,24 +36,33 @@
#[derive(Debug)]
pub(crate) struct OsStorage {
- ctrl_c: EventInfo,
ctrl_break: EventInfo,
+ ctrl_close: EventInfo,
+ ctrl_c: EventInfo,
+ ctrl_logoff: EventInfo,
+ ctrl_shutdown: EventInfo,
}
impl Init for OsStorage {
fn init() -> Self {
Self {
- ctrl_c: EventInfo::default(),
- ctrl_break: EventInfo::default(),
+ ctrl_break: Default::default(),
+ ctrl_close: Default::default(),
+ ctrl_c: Default::default(),
+ ctrl_logoff: Default::default(),
+ ctrl_shutdown: Default::default(),
}
}
}
impl Storage for OsStorage {
fn event_info(&self, id: EventId) -> Option<&EventInfo> {
- match DWORD::try_from(id) {
- Ok(CTRL_C_EVENT) => Some(&self.ctrl_c),
- Ok(CTRL_BREAK_EVENT) => Some(&self.ctrl_break),
+ match u32::try_from(id) {
+ Ok(console::CTRL_BREAK_EVENT) => Some(&self.ctrl_break),
+ Ok(console::CTRL_CLOSE_EVENT) => Some(&self.ctrl_close),
+ Ok(console::CTRL_C_EVENT) => Some(&self.ctrl_c),
+ Ok(console::CTRL_LOGOFF_EVENT) => Some(&self.ctrl_logoff),
+ Ok(console::CTRL_SHUTDOWN_EVENT) => Some(&self.ctrl_shutdown),
_ => None,
}
}
@@ -51,8 +71,11 @@
where
F: FnMut(&'a EventInfo),
{
- f(&self.ctrl_c);
f(&self.ctrl_break);
+ f(&self.ctrl_close);
+ f(&self.ctrl_c);
+ f(&self.ctrl_logoff);
+ f(&self.ctrl_shutdown);
}
}
@@ -71,7 +94,7 @@
let mut init = None;
INIT.call_once(|| unsafe {
- let rc = SetConsoleCtrlHandler(Some(handler), TRUE);
+ let rc = console::SetConsoleCtrlHandler(Some(handler), 1);
let ret = if rc == 0 {
Err(io::Error::last_os_error())
} else {
@@ -84,7 +107,7 @@
init.unwrap_or_else(|| Ok(()))
}
-unsafe extern "system" fn handler(ty: DWORD) -> BOOL {
+unsafe extern "system" fn handler(ty: u32) -> BOOL {
let globals = globals();
globals.record_event(ty as EventId);
@@ -93,11 +116,11 @@
// have the same restrictions as in Unix signal handlers, meaning we can
// go ahead and perform the broadcast here.
if globals.broadcast() {
- TRUE
+ 1
} else {
// No one is listening for this notification any more
// let the OS fire the next (possibly the default) handler.
- FALSE
+ 0
}
}
@@ -121,7 +144,7 @@
// like sending signals on Unix, so we'll stub out the actual OS
// integration and test that our handling works.
unsafe {
- super::handler(CTRL_C_EVENT);
+ super::handler(console::CTRL_C_EVENT);
}
assert_ready_ok!(ctrl_c.poll());
@@ -138,13 +161,67 @@
// like sending signals on Unix, so we'll stub out the actual OS
// integration and test that our handling works.
unsafe {
- super::handler(CTRL_BREAK_EVENT);
+ super::handler(console::CTRL_BREAK_EVENT);
}
ctrl_break.recv().await.unwrap();
});
}
+ #[test]
+ fn ctrl_close() {
+ let rt = rt();
+
+ rt.block_on(async {
+ let mut ctrl_close = assert_ok!(crate::signal::windows::ctrl_close());
+
+ // Windows doesn't have a good programmatic way of sending events
+ // like sending signals on Unix, so we'll stub out the actual OS
+ // integration and test that our handling works.
+ unsafe {
+ super::handler(console::CTRL_CLOSE_EVENT);
+ }
+
+ ctrl_close.recv().await.unwrap();
+ });
+ }
+
+ #[test]
+ fn ctrl_shutdown() {
+ let rt = rt();
+
+ rt.block_on(async {
+ let mut ctrl_shutdown = assert_ok!(crate::signal::windows::ctrl_shutdown());
+
+ // Windows doesn't have a good programmatic way of sending events
+ // like sending signals on Unix, so we'll stub out the actual OS
+ // integration and test that our handling works.
+ unsafe {
+ super::handler(console::CTRL_SHUTDOWN_EVENT);
+ }
+
+ ctrl_shutdown.recv().await.unwrap();
+ });
+ }
+
+ #[test]
+ fn ctrl_logoff() {
+ let rt = rt();
+
+ rt.block_on(async {
+ let mut ctrl_logoff = assert_ok!(crate::signal::windows::ctrl_logoff());
+
+ // Windows doesn't have a good programmatic way of sending events
+ // like sending signals on Unix, so we'll stub out the actual OS
+ // integration and test that our handling works.
+ unsafe {
+ super::handler(console::CTRL_LOGOFF_EVENT);
+ }
+
+ ctrl_logoff.recv().await.unwrap();
+ });
+ }
+
fn rt() -> Runtime {
crate::runtime::Builder::new_current_thread()
.build()
diff --git a/src/sync/barrier.rs b/src/sync/barrier.rs
index 0e39dac..2ce1d73 100644
--- a/src/sync/barrier.rs
+++ b/src/sync/barrier.rs
@@ -1,5 +1,7 @@
use crate::loom::sync::Mutex;
use crate::sync::watch;
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+use crate::util::trace;
/// A barrier enables multiple tasks to synchronize the beginning of some computation.
///
@@ -41,6 +43,8 @@
state: Mutex<BarrierState>,
wait: watch::Receiver<usize>,
n: usize,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
}
#[derive(Debug)]
@@ -55,6 +59,7 @@
///
/// A barrier will block `n`-1 tasks which call [`Barrier::wait`] and then wake up all
/// tasks at once when the `n`th task calls `wait`.
+ #[track_caller]
pub fn new(mut n: usize) -> Barrier {
let (waker, wait) = crate::sync::watch::channel(0);
@@ -65,6 +70,32 @@
n = 1;
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = std::panic::Location::caller();
+ let resource_span = tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "Barrier",
+ kind = "Sync",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ );
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ size = n,
+ );
+
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ arrived = 0,
+ )
+ });
+ resource_span
+ };
+
Barrier {
state: Mutex::new(BarrierState {
waker,
@@ -73,6 +104,8 @@
}),
n,
wait,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -85,10 +118,24 @@
/// [`BarrierWaitResult::is_leader`] when returning from this function, and all other tasks
/// will receive a result that will return `false` from `is_leader`.
pub async fn wait(&self) -> BarrierWaitResult {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ return trace::async_op(
+ || self.wait_internal(),
+ self.resource_span.clone(),
+ "Barrier::wait",
+ "poll",
+ false,
+ )
+ .await;
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ return self.wait_internal().await;
+ }
+ async fn wait_internal(&self) -> BarrierWaitResult {
// NOTE: we are taking a _synchronous_ lock here.
// It is okay to do so because the critical section is fast and never yields, so it cannot
// deadlock even if another future is concurrently holding the lock.
- // It is _desireable_ to do so as synchronous Mutexes are, at least in theory, faster than
+ // It is _desirable_ to do so as synchronous Mutexes are, at least in theory, faster than
// the asynchronous counter-parts, so we should use them where possible [citation needed].
// NOTE: the extra scope here is so that the compiler doesn't think `state` is held across
// a yield point, and thus marks the returned future as !Send.
@@ -96,7 +143,23 @@
let mut state = self.state.lock();
let generation = state.generation;
state.arrived += 1;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ arrived = 1,
+ arrived.op = "add",
+ );
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ tracing::trace!(
+ target: "runtime::resource::async_op::state_update",
+ arrived = true,
+ );
if state.arrived == self.n {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ tracing::trace!(
+ target: "runtime::resource::async_op::state_update",
+ is_leader = true,
+ );
// we are the leader for this generation
// wake everyone, increment the generation, and return
state
diff --git a/src/sync/batch_semaphore.rs b/src/sync/batch_semaphore.rs
index b5c39d2..57493f4 100644
--- a/src/sync/batch_semaphore.rs
+++ b/src/sync/batch_semaphore.rs
@@ -19,6 +19,8 @@
use crate::loom::sync::atomic::AtomicUsize;
use crate::loom::sync::{Mutex, MutexGuard};
use crate::util::linked_list::{self, LinkedList};
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+use crate::util::trace;
use crate::util::WakeList;
use std::future::Future;
@@ -35,6 +37,8 @@
waiters: Mutex<Waitlist>,
/// The current number of available permits in the semaphore.
permits: AtomicUsize,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
}
struct Waitlist {
@@ -45,7 +49,7 @@
/// Error returned from the [`Semaphore::try_acquire`] function.
///
/// [`Semaphore::try_acquire`]: crate::sync::Semaphore::try_acquire
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq)]
pub enum TryAcquireError {
/// The semaphore has been [closed] and cannot issue new permits.
///
@@ -101,10 +105,21 @@
/// use `UnsafeCell` internally.
pointers: linked_list::Pointers<Waiter>,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ ctx: trace::AsyncOpTracingCtx,
+
/// Should not be `Unpin`.
_p: PhantomPinned,
}
+generate_addr_of_methods! {
+ impl<> Waiter {
+ unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> {
+ &self.pointers
+ }
+ }
+}
+
impl Semaphore {
/// The maximum number of permits which a semaphore can hold.
///
@@ -129,12 +144,34 @@
"a semaphore may not have more than MAX_PERMITS permits ({})",
Self::MAX_PERMITS
);
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let resource_span = tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "Semaphore",
+ kind = "Sync",
+ is_internal = true
+ );
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ permits = permits,
+ permits.op = "override",
+ )
+ });
+ resource_span
+ };
+
Self {
permits: AtomicUsize::new(permits << Self::PERMIT_SHIFT),
waiters: Mutex::new(Waitlist {
queue: LinkedList::new(),
closed: false,
}),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -156,6 +193,8 @@
queue: LinkedList::new(),
closed: false,
}),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span::none(),
}
}
@@ -224,7 +263,10 @@
let next = curr - num_permits;
match self.permits.compare_exchange(curr, next, AcqRel, Acquire) {
- Ok(_) => return Ok(()),
+ Ok(_) => {
+ // TODO: Instrument once issue has been solved}
+ return Ok(());
+ }
Err(actual) => curr = actual,
}
}
@@ -283,6 +325,17 @@
rem,
Self::MAX_PERMITS
);
+
+ // add remaining permits back
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ permits = rem,
+ permits.op = "add",
+ )
+ });
+
rem = 0;
}
@@ -347,6 +400,20 @@
acquired += acq;
if remaining == 0 {
if !queued {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ permits = acquired,
+ permits.op = "sub",
+ );
+ tracing::trace!(
+ target: "runtime::resource::async_op::state_update",
+ permits_obtained = acquired,
+ permits.op = "add",
+ )
+ });
+
return Ready(Ok(()));
} else if lock.is_none() {
break self.waiters.lock();
@@ -362,6 +429,15 @@
return Ready(Err(AcquireError::closed()));
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ permits = acquired,
+ permits.op = "sub",
+ )
+ });
+
if node.assign_permits(&mut acquired) {
self.add_permits_locked(acquired, waiters);
return Ready(Ok(()));
@@ -406,11 +482,16 @@
}
impl Waiter {
- fn new(num_permits: u32) -> Self {
+ fn new(
+ num_permits: u32,
+ #[cfg(all(tokio_unstable, feature = "tracing"))] ctx: trace::AsyncOpTracingCtx,
+ ) -> Self {
Waiter {
waker: UnsafeCell::new(None),
state: AtomicUsize::new(num_permits as usize),
pointers: linked_list::Pointers::new(),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ ctx,
_p: PhantomPinned,
}
}
@@ -426,6 +507,14 @@
match self.state.compare_exchange(curr, next, AcqRel, Acquire) {
Ok(_) => {
*n -= assign;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.ctx.async_op_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::async_op::state_update",
+ permits_obtained = assign,
+ permits.op = "add",
+ );
+ });
return next == 0;
}
Err(actual) => curr = actual,
@@ -438,12 +527,26 @@
type Output = Result<(), AcquireError>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
- // First, ensure the current task has enough budget to proceed.
- let coop = ready!(crate::coop::poll_proceed(cx));
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _resource_span = self.node.ctx.resource_span.clone().entered();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _async_op_span = self.node.ctx.async_op_span.clone().entered();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _async_op_poll_span = self.node.ctx.async_op_poll_span.clone().entered();
let (node, semaphore, needed, queued) = self.project();
- match semaphore.poll_acquire(cx, needed, node, *queued) {
+ // First, ensure the current task has enough budget to proceed.
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let coop = ready!(trace_poll_op!(
+ "poll_acquire",
+ crate::runtime::coop::poll_proceed(cx),
+ ));
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
+
+ let result = match semaphore.poll_acquire(cx, needed, node, *queued) {
Pending => {
*queued = true;
Pending
@@ -454,18 +557,59 @@
*queued = false;
Ready(Ok(()))
}
- }
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ return trace_poll_op!("poll_acquire", result);
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ return result;
}
}
impl<'a> Acquire<'a> {
fn new(semaphore: &'a Semaphore, num_permits: u32) -> Self {
- Self {
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ return Self {
node: Waiter::new(num_permits),
semaphore,
num_permits,
queued: false,
- }
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ return semaphore.resource_span.in_scope(|| {
+ let async_op_span =
+ tracing::trace_span!("runtime.resource.async_op", source = "Acquire::new");
+ let async_op_poll_span = async_op_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::async_op::state_update",
+ permits_requested = num_permits,
+ permits.op = "override",
+ );
+
+ tracing::trace!(
+ target: "runtime::resource::async_op::state_update",
+ permits_obtained = 0usize,
+ permits.op = "override",
+ );
+
+ tracing::trace_span!("runtime.resource.async_op.poll")
+ });
+
+ let ctx = trace::AsyncOpTracingCtx {
+ async_op_span,
+ async_op_poll_span,
+ resource_span: semaphore.resource_span.clone(),
+ };
+
+ Self {
+ node: Waiter::new(num_permits, ctx),
+ semaphore,
+ num_permits,
+ queued: false,
+ }
+ });
}
fn project(self: Pin<&mut Self>) -> (Pin<&mut Waiter>, &Semaphore, u32, &mut bool) {
@@ -568,12 +712,6 @@
///
/// `Waiter` is forced to be !Unpin.
unsafe impl linked_list::Link for Waiter {
- // XXX: ideally, we would be able to use `Pin` here, to enforce the
- // invariant that list entries may not move while in the list. However, we
- // can't do this currently, as using `Pin<&'a mut Waiter>` as the `Handle`
- // type would require `Semaphore` to be generic over a lifetime. We can't
- // use `Pin<*mut Waiter>`, as raw pointers are `Unpin` regardless of whether
- // or not they dereference to an `!Unpin` target.
type Handle = NonNull<Waiter>;
type Target = Waiter;
@@ -585,7 +723,7 @@
ptr
}
- unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
- NonNull::from(&mut target.as_mut().pointers)
+ unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
+ Waiter::addr_of_pointers(target)
}
}
diff --git a/src/sync/broadcast.rs b/src/sync/broadcast.rs
index 0d9cd3b..1c6b2ca 100644
--- a/src/sync/broadcast.rs
+++ b/src/sync/broadcast.rs
@@ -18,6 +18,9 @@
//! returned [`Receiver`] will receive values sent **after** the call to
//! `subscribe`.
//!
+//! This channel is also suitable for the single-producer multi-consumer
+//! use-case, where a single sender broadcasts values to many receivers.
+//!
//! ## Lagging
//!
//! As sent messages must be retained until **all** [`Receiver`] handles receive
@@ -230,7 +233,7 @@
///
/// [`recv`]: crate::sync::broadcast::Receiver::recv
/// [`Receiver`]: crate::sync::broadcast::Receiver
- #[derive(Debug, PartialEq)]
+ #[derive(Debug, PartialEq, Eq, Clone)]
pub enum RecvError {
/// There are no more active senders implying no further messages will ever
/// be sent.
@@ -258,7 +261,7 @@
///
/// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv
/// [`Receiver`]: crate::sync::broadcast::Receiver
- #[derive(Debug, PartialEq)]
+ #[derive(Debug, PartialEq, Eq, Clone)]
pub enum TryRecvError {
/// The channel is currently empty. There are still active
/// [`Sender`] handles, so data may yet become available.
@@ -336,9 +339,6 @@
/// Uniquely identifies the `send` stored in the slot.
pos: u64,
- /// True signals the channel is closed.
- closed: bool,
-
/// The value being broadcast.
///
/// The value is set by `send` when the write lock is held. When a reader
@@ -361,6 +361,14 @@
_p: PhantomPinned,
}
+generate_addr_of_methods! {
+ impl<> Waiter {
+ unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> {
+ &self.pointers
+ }
+ }
+}
+
struct RecvGuard<'a, T> {
slot: RwLockReadGuard<'a, Slot<T>>,
}
@@ -425,6 +433,12 @@
/// tx.send(20).unwrap();
/// }
/// ```
+///
+/// # Panics
+///
+/// This will panic if `capacity` is equal to `0` or larger
+/// than `usize::MAX / 2`.
+#[track_caller]
pub fn channel<T: Clone>(mut capacity: usize) -> (Sender<T>, Receiver<T>) {
assert!(capacity > 0, "capacity is empty");
assert!(capacity <= usize::MAX >> 1, "requested capacity too large");
@@ -438,7 +452,6 @@
buffer.push(RwLock::new(Slot {
rem: AtomicUsize::new(0),
pos: (i as u64).wrapping_sub(capacity as u64),
- closed: false,
val: UnsafeCell::new(None),
}));
}
@@ -523,8 +536,43 @@
/// }
/// ```
pub fn send(&self, value: T) -> Result<usize, SendError<T>> {
- self.send2(Some(value))
- .map_err(|SendError(maybe_v)| SendError(maybe_v.unwrap()))
+ let mut tail = self.shared.tail.lock();
+
+ if tail.rx_cnt == 0 {
+ return Err(SendError(value));
+ }
+
+ // Position to write into
+ let pos = tail.pos;
+ let rem = tail.rx_cnt;
+ let idx = (pos & self.shared.mask as u64) as usize;
+
+ // Update the tail position
+ tail.pos = tail.pos.wrapping_add(1);
+
+ // Get the slot
+ let mut slot = self.shared.buffer[idx].write().unwrap();
+
+ // Track the position
+ slot.pos = pos;
+
+ // Set remaining receivers
+ slot.rem.with_mut(|v| *v = rem);
+
+ // Write the value
+ slot.val = UnsafeCell::new(Some(value));
+
+ // Release the slot lock before notifying the receivers.
+ drop(slot);
+
+ tail.notify_rx();
+
+ // Release the mutex. This must happen after the slot lock is released,
+ // otherwise the writer lock bit could be cleared while another thread
+ // is in the critical section.
+ drop(tail);
+
+ Ok(rem)
}
/// Creates a new [`Receiver`] handle that will receive values sent **after**
@@ -555,6 +603,97 @@
new_receiver(shared)
}
+ /// Returns the number of queued values.
+ ///
+ /// A value is queued until it has either been seen by all receivers that were alive at the time
+ /// it was sent, or has been evicted from the queue by subsequent sends that exceeded the
+ /// queue's capacity.
+ ///
+ /// # Note
+ ///
+ /// In contrast to [`Receiver::len`], this method only reports queued values and not values that
+ /// have been evicted from the queue before being seen by all receivers.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::broadcast;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx1) = broadcast::channel(16);
+ /// let mut rx2 = tx.subscribe();
+ ///
+ /// tx.send(10).unwrap();
+ /// tx.send(20).unwrap();
+ /// tx.send(30).unwrap();
+ ///
+ /// assert_eq!(tx.len(), 3);
+ ///
+ /// rx1.recv().await.unwrap();
+ ///
+ /// // The len is still 3 since rx2 hasn't seen the first value yet.
+ /// assert_eq!(tx.len(), 3);
+ ///
+ /// rx2.recv().await.unwrap();
+ ///
+ /// assert_eq!(tx.len(), 2);
+ /// }
+ /// ```
+ pub fn len(&self) -> usize {
+ let tail = self.shared.tail.lock();
+
+ let base_idx = (tail.pos & self.shared.mask as u64) as usize;
+ let mut low = 0;
+ let mut high = self.shared.buffer.len();
+ while low < high {
+ let mid = low + (high - low) / 2;
+ let idx = base_idx.wrapping_add(mid) & self.shared.mask;
+ if self.shared.buffer[idx].read().unwrap().rem.load(SeqCst) == 0 {
+ low = mid + 1;
+ } else {
+ high = mid;
+ }
+ }
+
+ self.shared.buffer.len() - low
+ }
+
+ /// Returns true if there are no queued values.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::broadcast;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx1) = broadcast::channel(16);
+ /// let mut rx2 = tx.subscribe();
+ ///
+ /// assert!(tx.is_empty());
+ ///
+ /// tx.send(10).unwrap();
+ ///
+ /// assert!(!tx.is_empty());
+ ///
+ /// rx1.recv().await.unwrap();
+ ///
+ /// // The queue is still not empty since rx2 hasn't seen the value.
+ /// assert!(!tx.is_empty());
+ ///
+ /// rx2.recv().await.unwrap();
+ ///
+ /// assert!(tx.is_empty());
+ /// }
+ /// ```
+ pub fn is_empty(&self) -> bool {
+ let tail = self.shared.tail.lock();
+
+ let idx = (tail.pos.wrapping_sub(1) & self.shared.mask as u64) as usize;
+ self.shared.buffer[idx].read().unwrap().rem.load(SeqCst) == 0
+ }
+
/// Returns the number of active receivers
///
/// An active receiver is a [`Receiver`] handle returned from [`channel`] or
@@ -596,52 +735,15 @@
tail.rx_cnt
}
- fn send2(&self, value: Option<T>) -> Result<usize, SendError<Option<T>>> {
+ fn close_channel(&self) {
let mut tail = self.shared.tail.lock();
-
- if tail.rx_cnt == 0 {
- return Err(SendError(value));
- }
-
- // Position to write into
- let pos = tail.pos;
- let rem = tail.rx_cnt;
- let idx = (pos & self.shared.mask as u64) as usize;
-
- // Update the tail position
- tail.pos = tail.pos.wrapping_add(1);
-
- // Get the slot
- let mut slot = self.shared.buffer[idx].write().unwrap();
-
- // Track the position
- slot.pos = pos;
-
- // Set remaining receivers
- slot.rem.with_mut(|v| *v = rem);
-
- // Set the closed bit if the value is `None`; otherwise write the value
- if value.is_none() {
- tail.closed = true;
- slot.closed = true;
- } else {
- slot.val.with_mut(|ptr| unsafe { *ptr = value });
- }
-
- // Release the slot lock before notifying the receivers.
- drop(slot);
+ tail.closed = true;
tail.notify_rx();
-
- // Release the mutex. This must happen after the slot lock is released,
- // otherwise the writer lock bit could be cleared while another thread
- // is in the critical section.
- drop(tail);
-
- Ok(rem)
}
}
+/// Create a new `Receiver` which reads starting from the tail.
fn new_receiver<T>(shared: Arc<Shared<T>>) -> Receiver<T> {
let mut tail = shared.tail.lock();
@@ -685,12 +787,79 @@
impl<T> Drop for Sender<T> {
fn drop(&mut self) {
if 1 == self.shared.num_tx.fetch_sub(1, SeqCst) {
- let _ = self.send2(None);
+ self.close_channel();
}
}
}
impl<T> Receiver<T> {
+ /// Returns the number of messages that were sent into the channel and that
+ /// this [`Receiver`] has yet to receive.
+ ///
+ /// If the returned value from `len` is larger than the next largest power of 2
+ /// of the capacity of the channel any call to [`recv`] will return an
+ /// `Err(RecvError::Lagged)` and any call to [`try_recv`] will return an
+ /// `Err(TryRecvError::Lagged)`, e.g. if the capacity of the channel is 10,
+ /// [`recv`] will start to return `Err(RecvError::Lagged)` once `len` returns
+ /// values larger than 16.
+ ///
+ /// [`Receiver`]: crate::sync::broadcast::Receiver
+ /// [`recv`]: crate::sync::broadcast::Receiver::recv
+ /// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::broadcast;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx1) = broadcast::channel(16);
+ ///
+ /// tx.send(10).unwrap();
+ /// tx.send(20).unwrap();
+ ///
+ /// assert_eq!(rx1.len(), 2);
+ /// assert_eq!(rx1.recv().await.unwrap(), 10);
+ /// assert_eq!(rx1.len(), 1);
+ /// assert_eq!(rx1.recv().await.unwrap(), 20);
+ /// assert_eq!(rx1.len(), 0);
+ /// }
+ /// ```
+ pub fn len(&self) -> usize {
+ let next_send_pos = self.shared.tail.lock().pos;
+ (next_send_pos - self.next) as usize
+ }
+
+ /// Returns true if there aren't any messages in the channel that the [`Receiver`]
+ /// has yet to receive.
+ ///
+ /// [`Receiver]: create::sync::broadcast::Receiver
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::broadcast;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx1) = broadcast::channel(16);
+ ///
+ /// assert!(rx1.is_empty());
+ ///
+ /// tx.send(10).unwrap();
+ /// tx.send(20).unwrap();
+ ///
+ /// assert!(!rx1.is_empty());
+ /// assert_eq!(rx1.recv().await.unwrap(), 10);
+ /// assert_eq!(rx1.recv().await.unwrap(), 20);
+ /// assert!(rx1.is_empty());
+ /// }
+ /// ```
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
/// Locks the next value if there is one.
fn recv_ref(
&mut self,
@@ -702,14 +871,6 @@
let mut slot = self.shared.buffer[idx].read().unwrap();
if slot.pos != self.next {
- let next_pos = slot.pos.wrapping_add(self.shared.buffer.len() as u64);
-
- // The receiver has read all current values in the channel and there
- // is no waiter to register
- if waiter.is_none() && next_pos == self.next {
- return Err(TryRecvError::Empty);
- }
-
// Release the `slot` lock before attempting to acquire the `tail`
// lock. This is required because `send2` acquires the tail lock
// first followed by the slot lock. Acquiring the locks in reverse
@@ -731,6 +892,13 @@
let next_pos = slot.pos.wrapping_add(self.shared.buffer.len() as u64);
if next_pos == self.next {
+ // At this point the channel is empty for *this* receiver. If
+ // it's been closed, then that's what we return, otherwise we
+ // set a waker and return empty.
+ if tail.closed {
+ return Err(TryRecvError::Closed);
+ }
+
// Store the waker
if let Some((waiter, waker)) = waiter {
// Safety: called while locked.
@@ -764,22 +932,7 @@
// catch up by skipping dropped messages and setting the
// internal cursor to the **oldest** message stored by the
// channel.
- //
- // However, finding the oldest position is a bit more
- // complicated than `tail-position - buffer-size`. When
- // the channel is closed, the tail position is incremented to
- // signal a new `None` message, but `None` is not stored in the
- // channel itself (see issue #2425 for why).
- //
- // To account for this, if the channel is closed, the tail
- // position is decremented by `buffer-size + 1`.
- let mut adjust = 0;
- if tail.closed {
- adjust = 1
- }
- let next = tail
- .pos
- .wrapping_sub(self.shared.buffer.len() as u64 + adjust);
+ let next = tail.pos.wrapping_sub(self.shared.buffer.len() as u64);
let missed = next.wrapping_sub(self.next);
@@ -800,15 +953,38 @@
self.next = self.next.wrapping_add(1);
- if slot.closed {
- return Err(TryRecvError::Closed);
- }
-
Ok(RecvGuard { slot })
}
}
impl<T: Clone> Receiver<T> {
+ /// Re-subscribes to the channel starting from the current tail element.
+ ///
+ /// This [`Receiver`] handle will receive a clone of all values sent
+ /// **after** it has resubscribed. This will not include elements that are
+ /// in the queue of the current receiver. Consider the following example.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::broadcast;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx) = broadcast::channel(2);
+ ///
+ /// tx.send(1).unwrap();
+ /// let mut rx2 = rx.resubscribe();
+ /// tx.send(2).unwrap();
+ ///
+ /// assert_eq!(rx2.recv().await.unwrap(), 2);
+ /// assert_eq!(rx.recv().await.unwrap(), 1);
+ /// }
+ /// ```
+ pub fn resubscribe(&self) -> Self {
+ let shared = self.shared.clone();
+ new_receiver(shared)
+ }
/// Receives the next value for this receiver.
///
/// Each [`Receiver`] handle will receive a clone of all values sent
@@ -1039,8 +1215,8 @@
ptr
}
- unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
- NonNull::from(&mut target.as_mut().pointers)
+ unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
+ Waiter::addr_of_pointers(target)
}
}
diff --git a/src/sync/mod.rs b/src/sync/mod.rs
index 457e6ab..8fba196 100644
--- a/src/sync/mod.rs
+++ b/src/sync/mod.rs
@@ -94,6 +94,10 @@
//! producers to a single consumer. This channel is often used to send work to a
//! task or to receive the result of many computations.
//!
+//! This is also the channel you should use if you want to send many messages
+//! from a single producer to a single consumer. There is no dedicated spsc
+//! channel.
+//!
//! **Example:** using an mpsc to incrementally stream the results of a series
//! of computations.
//!
@@ -244,6 +248,10 @@
//! This channel tends to be used less often than `oneshot` and `mpsc` but still
//! has its use cases.
//!
+//! This is also the channel you should use if you want to broadcast values from
+//! a single producer to many consumers. There is no dedicated spmc broadcast
+//! channel.
+//!
//! Basic usage
//!
//! ```
diff --git a/src/sync/mpsc/block.rs b/src/sync/mpsc/block.rs
index 58f4a9f..39c3e1b 100644
--- a/src/sync/mpsc/block.rs
+++ b/src/sync/mpsc/block.rs
@@ -1,6 +1,7 @@
use crate::loom::cell::UnsafeCell;
use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize};
+use std::alloc::Layout;
use std::mem::MaybeUninit;
use std::ops;
use std::ptr::{self, NonNull};
@@ -10,6 +11,17 @@
///
/// Each block in the list can hold up to `BLOCK_CAP` messages.
pub(crate) struct Block<T> {
+ /// The header fields.
+ header: BlockHeader<T>,
+
+ /// Array containing values pushed into the block. Values are stored in a
+ /// continuous array in order to improve cache line behavior when reading.
+ /// The values must be manually dropped.
+ values: Values<T>,
+}
+
+/// Extra fields for a `Block<T>`.
+struct BlockHeader<T> {
/// The start index of this block.
///
/// Slots in this block have indices in `start_index .. start_index + BLOCK_CAP`.
@@ -24,11 +36,6 @@
/// The observed `tail_position` value *after* the block has been passed by
/// `block_tail`.
observed_tail_position: UnsafeCell<usize>,
-
- /// Array containing values pushed into the block. Values are stored in a
- /// continuous array in order to improve cache line behavior when reading.
- /// The values must be manually dropped.
- values: Values<T>,
}
pub(crate) enum Read<T> {
@@ -36,6 +43,7 @@
Closed,
}
+#[repr(transparent)]
struct Values<T>([UnsafeCell<MaybeUninit<T>>; BLOCK_CAP]);
use super::BLOCK_CAP;
@@ -71,28 +79,56 @@
SLOT_MASK & slot_index
}
+generate_addr_of_methods! {
+ impl<T> Block<T> {
+ unsafe fn addr_of_header(self: NonNull<Self>) -> NonNull<BlockHeader<T>> {
+ &self.header
+ }
+
+ unsafe fn addr_of_values(self: NonNull<Self>) -> NonNull<Values<T>> {
+ &self.values
+ }
+ }
+}
+
impl<T> Block<T> {
- pub(crate) fn new(start_index: usize) -> Block<T> {
- Block {
- // The absolute index in the channel of the first slot in the block.
- start_index,
+ pub(crate) fn new(start_index: usize) -> Box<Block<T>> {
+ unsafe {
+ // Allocate the block on the heap.
+ // SAFETY: The size of the Block<T> is non-zero, since it is at least the size of the header.
+ let block = std::alloc::alloc(Layout::new::<Block<T>>()) as *mut Block<T>;
+ let block = match NonNull::new(block) {
+ Some(block) => block,
+ None => std::alloc::handle_alloc_error(Layout::new::<Block<T>>()),
+ };
- // Pointer to the next block in the linked list.
- next: AtomicPtr::new(ptr::null_mut()),
+ // Write the header to the block.
+ Block::addr_of_header(block).as_ptr().write(BlockHeader {
+ // The absolute index in the channel of the first slot in the block.
+ start_index,
- ready_slots: AtomicUsize::new(0),
+ // Pointer to the next block in the linked list.
+ next: AtomicPtr::new(ptr::null_mut()),
- observed_tail_position: UnsafeCell::new(0),
+ ready_slots: AtomicUsize::new(0),
- // Value storage
- values: unsafe { Values::uninitialized() },
+ observed_tail_position: UnsafeCell::new(0),
+ });
+
+ // Initialize the values array.
+ Values::initialize(Block::addr_of_values(block));
+
+ // Convert the pointer to a `Box`.
+ // Safety: The raw pointer was allocated using the global allocator, and with
+ // the layout for a `Block<T>`, so it's valid to convert it to box.
+ Box::from_raw(block.as_ptr())
}
}
/// Returns `true` if the block matches the given index.
pub(crate) fn is_at_index(&self, index: usize) -> bool {
debug_assert!(offset(index) == 0);
- self.start_index == index
+ self.header.start_index == index
}
/// Returns the number of blocks between `self` and the block at the
@@ -101,7 +137,7 @@
/// `start_index` must represent a block *after* `self`.
pub(crate) fn distance(&self, other_index: usize) -> usize {
debug_assert!(offset(other_index) == 0);
- other_index.wrapping_sub(self.start_index) / BLOCK_CAP
+ other_index.wrapping_sub(self.header.start_index) / BLOCK_CAP
}
/// Reads the value at the given offset.
@@ -116,7 +152,7 @@
pub(crate) unsafe fn read(&self, slot_index: usize) -> Option<Read<T>> {
let offset = offset(slot_index);
- let ready_bits = self.ready_slots.load(Acquire);
+ let ready_bits = self.header.ready_slots.load(Acquire);
if !is_ready(ready_bits, offset) {
if is_tx_closed(ready_bits) {
@@ -156,7 +192,7 @@
/// Signal to the receiver that the sender half of the list is closed.
pub(crate) unsafe fn tx_close(&self) {
- self.ready_slots.fetch_or(TX_CLOSED, Release);
+ self.header.ready_slots.fetch_or(TX_CLOSED, Release);
}
/// Resets the block to a blank state. This enables reusing blocks in the
@@ -169,9 +205,9 @@
/// * All slots are empty.
/// * The caller holds a unique pointer to the block.
pub(crate) unsafe fn reclaim(&mut self) {
- self.start_index = 0;
- self.next = AtomicPtr::new(ptr::null_mut());
- self.ready_slots = AtomicUsize::new(0);
+ self.header.start_index = 0;
+ self.header.next = AtomicPtr::new(ptr::null_mut());
+ self.header.ready_slots = AtomicUsize::new(0);
}
/// Releases the block to the rx half for freeing.
@@ -187,19 +223,20 @@
pub(crate) unsafe fn tx_release(&self, tail_position: usize) {
// Track the observed tail_position. Any sender targeting a greater
// tail_position is guaranteed to not access this block.
- self.observed_tail_position
+ self.header
+ .observed_tail_position
.with_mut(|ptr| *ptr = tail_position);
// Set the released bit, signalling to the receiver that it is safe to
// free the block's memory as soon as all slots **prior** to
// `observed_tail_position` have been filled.
- self.ready_slots.fetch_or(RELEASED, Release);
+ self.header.ready_slots.fetch_or(RELEASED, Release);
}
/// Mark a slot as ready
fn set_ready(&self, slot: usize) {
let mask = 1 << slot;
- self.ready_slots.fetch_or(mask, Release);
+ self.header.ready_slots.fetch_or(mask, Release);
}
/// Returns `true` when all slots have their `ready` bits set.
@@ -214,25 +251,31 @@
/// single atomic cell. However, this could have negative impact on cache
/// behavior as there would be many more mutations to a single slot.
pub(crate) fn is_final(&self) -> bool {
- self.ready_slots.load(Acquire) & READY_MASK == READY_MASK
+ self.header.ready_slots.load(Acquire) & READY_MASK == READY_MASK
}
/// Returns the `observed_tail_position` value, if set
pub(crate) fn observed_tail_position(&self) -> Option<usize> {
- if 0 == RELEASED & self.ready_slots.load(Acquire) {
+ if 0 == RELEASED & self.header.ready_slots.load(Acquire) {
None
} else {
- Some(self.observed_tail_position.with(|ptr| unsafe { *ptr }))
+ Some(
+ self.header
+ .observed_tail_position
+ .with(|ptr| unsafe { *ptr }),
+ )
}
}
/// Loads the next block
pub(crate) fn load_next(&self, ordering: Ordering) -> Option<NonNull<Block<T>>> {
- let ret = NonNull::new(self.next.load(ordering));
+ let ret = NonNull::new(self.header.next.load(ordering));
debug_assert!(unsafe {
- ret.map(|block| block.as_ref().start_index == self.start_index.wrapping_add(BLOCK_CAP))
- .unwrap_or(true)
+ ret.map(|block| {
+ block.as_ref().header.start_index == self.header.start_index.wrapping_add(BLOCK_CAP)
+ })
+ .unwrap_or(true)
});
ret
@@ -260,9 +303,10 @@
success: Ordering,
failure: Ordering,
) -> Result<(), NonNull<Block<T>>> {
- block.as_mut().start_index = self.start_index.wrapping_add(BLOCK_CAP);
+ block.as_mut().header.start_index = self.header.start_index.wrapping_add(BLOCK_CAP);
let next_ptr = self
+ .header
.next
.compare_exchange(ptr::null_mut(), block.as_ptr(), success, failure)
.unwrap_or_else(|x| x);
@@ -291,7 +335,7 @@
// Create the new block. It is assumed that the block will become the
// next one after `&self`. If this turns out to not be the case,
// `start_index` is updated accordingly.
- let new_block = Box::new(Block::new(self.start_index + BLOCK_CAP));
+ let new_block = Block::new(self.header.start_index + BLOCK_CAP);
let mut new_block = unsafe { NonNull::new_unchecked(Box::into_raw(new_block)) };
@@ -308,7 +352,8 @@
// `Release` ensures that the newly allocated block is available to
// other threads acquiring the next pointer.
let next = NonNull::new(
- self.next
+ self.header
+ .next
.compare_exchange(ptr::null_mut(), new_block.as_ptr(), AcqRel, Acquire)
.unwrap_or_else(|x| x),
);
@@ -360,19 +405,20 @@
}
impl<T> Values<T> {
- unsafe fn uninitialized() -> Values<T> {
- let mut vals = MaybeUninit::uninit();
-
+ /// Initialize a `Values` struct from a pointer.
+ ///
+ /// # Safety
+ ///
+ /// The raw pointer must be valid for writing a `Values<T>`.
+ unsafe fn initialize(_value: NonNull<Values<T>>) {
// When fuzzing, `UnsafeCell` needs to be initialized.
if_loom! {
- let p = vals.as_mut_ptr() as *mut UnsafeCell<MaybeUninit<T>>;
+ let p = _value.as_ptr() as *mut UnsafeCell<MaybeUninit<T>>;
for i in 0..BLOCK_CAP {
p.add(i)
.write(UnsafeCell::new(MaybeUninit::uninit()));
}
}
-
- Values(vals.assume_init())
}
}
@@ -383,3 +429,20 @@
self.0.index(index)
}
}
+
+#[cfg(all(test, not(loom)))]
+#[test]
+fn assert_no_stack_overflow() {
+ // https://github.com/tokio-rs/tokio/issues/5293
+
+ struct Foo {
+ _a: [u8; 2_000_000],
+ }
+
+ assert_eq!(
+ Layout::new::<MaybeUninit<Block<Foo>>>(),
+ Layout::new::<Block<Foo>>()
+ );
+
+ let _block = Block::<Foo>::new(0);
+}
diff --git a/src/sync/mpsc/bounded.rs b/src/sync/mpsc/bounded.rs
index 5a2bfa6..8babdc7 100644
--- a/src/sync/mpsc/bounded.rs
+++ b/src/sync/mpsc/bounded.rs
@@ -1,3 +1,4 @@
+use crate::loom::sync::Arc;
use crate::sync::batch_semaphore::{self as semaphore, TryAcquireError};
use crate::sync::mpsc::chan;
use crate::sync::mpsc::error::{SendError, TryRecvError, TrySendError};
@@ -22,6 +23,40 @@
chan: chan::Tx<T, Semaphore>,
}
+/// A sender that does not prevent the channel from being closed.
+///
+/// If all [`Sender`] instances of a channel were dropped and only `WeakSender`
+/// instances remain, the channel is closed.
+///
+/// In order to send messages, the `WeakSender` needs to be upgraded using
+/// [`WeakSender::upgrade`], which returns `Option<Sender>`. It returns `None`
+/// if all `Sender`s have been dropped, and otherwise it returns a `Sender`.
+///
+/// [`Sender`]: Sender
+/// [`WeakSender::upgrade`]: WeakSender::upgrade
+///
+/// #Examples
+///
+/// ```
+/// use tokio::sync::mpsc::channel;
+///
+/// #[tokio::main]
+/// async fn main() {
+/// let (tx, _rx) = channel::<i32>(15);
+/// let tx_weak = tx.downgrade();
+///
+/// // Upgrading will succeed because `tx` still exists.
+/// assert!(tx_weak.upgrade().is_some());
+///
+/// // If we drop `tx`, then it will fail.
+/// drop(tx);
+/// assert!(tx_weak.clone().upgrade().is_none());
+/// }
+/// ```
+pub struct WeakSender<T> {
+ chan: Arc<chan::Chan<T, Semaphore>>,
+}
+
/// Permits to send one value into the channel.
///
/// `Permit` values are returned by [`Sender::reserve()`] and [`Sender::try_reserve()`]
@@ -105,9 +140,13 @@
/// }
/// }
/// ```
+#[track_caller]
pub fn channel<T>(buffer: usize) -> (Sender<T>, Receiver<T>) {
assert!(buffer > 0, "mpsc bounded channel requires buffer > 0");
- let semaphore = (semaphore::Semaphore::new(buffer), buffer);
+ let semaphore = Semaphore {
+ semaphore: semaphore::Semaphore::new(buffer),
+ bound: buffer,
+ };
let (tx, rx) = chan::channel(semaphore);
let tx = Sender::new(tx);
@@ -118,7 +157,11 @@
/// Channel semaphore is a tuple of the semaphore implementation and a `usize`
/// representing the channel bound.
-type Semaphore = (semaphore::Semaphore, usize);
+#[derive(Debug)]
+pub(crate) struct Semaphore {
+ pub(crate) semaphore: semaphore::Semaphore,
+ pub(crate) bound: usize,
+}
impl<T> Receiver<T> {
pub(crate) fn new(chan: chan::Rx<T, Semaphore>) -> Receiver<T> {
@@ -281,6 +324,7 @@
/// sync_code.join().unwrap()
/// }
/// ```
+ #[track_caller]
#[cfg(feature = "sync")]
pub fn blocking_recv(&mut self) -> Option<T> {
crate::future::block_on(self.recv())
@@ -535,7 +579,7 @@
/// }
/// ```
pub fn try_send(&self, message: T) -> Result<(), TrySendError<T>> {
- match self.chan.semaphore().0.try_acquire(1) {
+ match self.chan.semaphore().semaphore.try_acquire(1) {
Ok(_) => {}
Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(message)),
Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(message)),
@@ -563,6 +607,11 @@
/// [`close`]: Receiver::close
/// [`Receiver`]: Receiver
///
+ /// # Panics
+ ///
+ /// This function panics if it is called outside the context of a Tokio
+ /// runtime [with time enabled](crate::runtime::Builder::enable_time).
+ ///
/// # Examples
///
/// In the following example, each call to `send_timeout` will block until the
@@ -645,6 +694,7 @@
/// sync_code.join().unwrap()
/// }
/// ```
+ #[track_caller]
#[cfg(feature = "sync")]
pub fn blocking_send(&self, value: T) -> Result<(), SendError<T>> {
crate::future::block_on(self.send(value))
@@ -809,7 +859,7 @@
}
async fn reserve_inner(&self) -> Result<(), SendError<()>> {
- match self.chan.semaphore().0.acquire(1).await {
+ match self.chan.semaphore().semaphore.acquire(1).await {
Ok(_) => Ok(()),
Err(_) => Err(SendError(())),
}
@@ -859,7 +909,7 @@
/// }
/// ```
pub fn try_reserve(&self) -> Result<Permit<'_, T>, TrySendError<()>> {
- match self.chan.semaphore().0.try_acquire(1) {
+ match self.chan.semaphore().semaphore.try_acquire(1) {
Ok(_) => {}
Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(())),
Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(())),
@@ -924,7 +974,7 @@
/// }
/// ```
pub fn try_reserve_owned(self) -> Result<OwnedPermit<T>, TrySendError<Self>> {
- match self.chan.semaphore().0.try_acquire(1) {
+ match self.chan.semaphore().semaphore.try_acquire(1) {
Ok(_) => {}
Err(TryAcquireError::Closed) => return Err(TrySendError::Closed(self)),
Err(TryAcquireError::NoPermits) => return Err(TrySendError::Full(self)),
@@ -955,6 +1005,8 @@
///
/// The capacity goes down when sending a value by calling [`send`] or by reserving capacity
/// with [`reserve`]. The capacity goes up when values are received by the [`Receiver`].
+ /// This is distinct from [`max_capacity`], which always returns buffer capacity initially
+ /// specified when calling [`channel`]
///
/// # Examples
///
@@ -980,8 +1032,56 @@
///
/// [`send`]: Sender::send
/// [`reserve`]: Sender::reserve
+ /// [`channel`]: channel
+ /// [`max_capacity`]: Sender::max_capacity
pub fn capacity(&self) -> usize {
- self.chan.semaphore().0.available_permits()
+ self.chan.semaphore().semaphore.available_permits()
+ }
+
+ /// Converts the `Sender` to a [`WeakSender`] that does not count
+ /// towards RAII semantics, i.e. if all `Sender` instances of the
+ /// channel were dropped and only `WeakSender` instances remain,
+ /// the channel is closed.
+ pub fn downgrade(&self) -> WeakSender<T> {
+ WeakSender {
+ chan: self.chan.downgrade(),
+ }
+ }
+
+ /// Returns the maximum buffer capacity of the channel.
+ ///
+ /// The maximum capacity is the buffer capacity initially specified when calling
+ /// [`channel`]. This is distinct from [`capacity`], which returns the *current*
+ /// available buffer capacity: as messages are sent and received, the
+ /// value returned by [`capacity`] will go up or down, whereas the value
+ /// returned by `max_capacity` will remain constant.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::mpsc;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, _rx) = mpsc::channel::<()>(5);
+ ///
+ /// // both max capacity and capacity are the same at first
+ /// assert_eq!(tx.max_capacity(), 5);
+ /// assert_eq!(tx.capacity(), 5);
+ ///
+ /// // Making a reservation doesn't change the max capacity.
+ /// let permit = tx.reserve().await.unwrap();
+ /// assert_eq!(tx.max_capacity(), 5);
+ /// // but drops the capacity by one
+ /// assert_eq!(tx.capacity(), 4);
+ /// }
+ /// ```
+ ///
+ /// [`channel`]: channel
+ /// [`max_capacity`]: Sender::max_capacity
+ /// [`capacity`]: Sender::capacity
+ pub fn max_capacity(&self) -> usize {
+ self.chan.semaphore().bound
}
}
@@ -1001,6 +1101,29 @@
}
}
+impl<T> Clone for WeakSender<T> {
+ fn clone(&self) -> Self {
+ WeakSender {
+ chan: self.chan.clone(),
+ }
+ }
+}
+
+impl<T> WeakSender<T> {
+ /// Tries to convert a WeakSender into a [`Sender`]. This will return `Some`
+ /// if there are other `Sender` instances alive and the channel wasn't
+ /// previously dropped, otherwise `None` is returned.
+ pub fn upgrade(&self) -> Option<Sender<T>> {
+ chan::Tx::upgrade(self.chan.clone()).map(Sender::new)
+ }
+}
+
+impl<T> fmt::Debug for WeakSender<T> {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt.debug_struct("WeakSender").finish()
+ }
+}
+
// ===== impl Permit =====
impl<T> Permit<'_, T> {
diff --git a/src/sync/mpsc/chan.rs b/src/sync/mpsc/chan.rs
index c3007de..edd3e95 100644
--- a/src/sync/mpsc/chan.rs
+++ b/src/sync/mpsc/chan.rs
@@ -2,15 +2,14 @@
use crate::loom::future::AtomicWaker;
use crate::loom::sync::atomic::AtomicUsize;
use crate::loom::sync::Arc;
-use crate::park::thread::CachedParkThread;
-use crate::park::Park;
+use crate::runtime::park::CachedParkThread;
use crate::sync::mpsc::error::TryRecvError;
-use crate::sync::mpsc::list;
+use crate::sync::mpsc::{bounded, list, unbounded};
use crate::sync::notify::Notify;
use std::fmt;
use std::process;
-use std::sync::atomic::Ordering::{AcqRel, Relaxed};
+use std::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release};
use std::task::Poll::{Pending, Ready};
use std::task::{Context, Poll};
@@ -46,7 +45,7 @@
fn is_closed(&self) -> bool;
}
-struct Chan<T, S> {
+pub(super) struct Chan<T, S> {
/// Notifies all tasks listening for the receiver being dropped.
notify_rx_closed: Notify,
@@ -129,6 +128,30 @@
Tx { inner: chan }
}
+ pub(super) fn downgrade(&self) -> Arc<Chan<T, S>> {
+ self.inner.clone()
+ }
+
+ // Returns the upgraded channel or None if the upgrade failed.
+ pub(super) fn upgrade(chan: Arc<Chan<T, S>>) -> Option<Self> {
+ let mut tx_count = chan.tx_count.load(Acquire);
+
+ loop {
+ if tx_count == 0 {
+ // channel is closed
+ return None;
+ }
+
+ match chan
+ .tx_count
+ .compare_exchange_weak(tx_count, tx_count + 1, AcqRel, Acquire)
+ {
+ Ok(_) => return Some(Tx { inner: chan }),
+ Err(prev_count) => tx_count = prev_count,
+ }
+ }
+ }
+
pub(super) fn semaphore(&self) -> &S {
&self.inner.semaphore
}
@@ -220,7 +243,7 @@
use super::block::Read::*;
// Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
self.inner.rx_fields.with_mut(|rx_fields_ptr| {
let rx_fields = unsafe { &mut *rx_fields_ptr };
@@ -301,13 +324,13 @@
// Park the thread until the problematic send has completed.
let mut park = CachedParkThread::new();
- let waker = park.unpark().into_waker();
+ let waker = park.waker().unwrap();
loop {
self.inner.rx_waker.register_by_ref(&waker);
// It is possible that the problematic send has now completed,
// so we have to check for messages again.
try_recv!();
- park.park().expect("park failed");
+ park.park();
}
})
}
@@ -345,7 +368,7 @@
fn drop(&mut self) {
use super::block::Read::Value;
- // Safety: the only owner of the rx fields is Chan, and eing
+ // Safety: the only owner of the rx fields is Chan, and being
// inside its own Drop means we're the last ones to touch it.
self.rx_fields.with_mut(|rx_fields_ptr| {
let rx_fields = unsafe { &mut *rx_fields_ptr };
@@ -358,32 +381,29 @@
// ===== impl Semaphore for (::Semaphore, capacity) =====
-impl Semaphore for (crate::sync::batch_semaphore::Semaphore, usize) {
+impl Semaphore for bounded::Semaphore {
fn add_permit(&self) {
- self.0.release(1)
+ self.semaphore.release(1)
}
fn is_idle(&self) -> bool {
- self.0.available_permits() == self.1
+ self.semaphore.available_permits() == self.bound
}
fn close(&self) {
- self.0.close();
+ self.semaphore.close();
}
fn is_closed(&self) -> bool {
- self.0.is_closed()
+ self.semaphore.is_closed()
}
}
// ===== impl Semaphore for AtomicUsize =====
-use std::sync::atomic::Ordering::{Acquire, Release};
-use std::usize;
-
-impl Semaphore for AtomicUsize {
+impl Semaphore for unbounded::Semaphore {
fn add_permit(&self) {
- let prev = self.fetch_sub(2, Release);
+ let prev = self.0.fetch_sub(2, Release);
if prev >> 1 == 0 {
// Something went wrong
@@ -392,14 +412,14 @@
}
fn is_idle(&self) -> bool {
- self.load(Acquire) >> 1 == 0
+ self.0.load(Acquire) >> 1 == 0
}
fn close(&self) {
- self.fetch_or(1, Release);
+ self.0.fetch_or(1, Release);
}
fn is_closed(&self) -> bool {
- self.load(Acquire) & 1 == 1
+ self.0.load(Acquire) & 1 == 1
}
}
diff --git a/src/sync/mpsc/error.rs b/src/sync/mpsc/error.rs
index b7b9cf7..1c789da 100644
--- a/src/sync/mpsc/error.rs
+++ b/src/sync/mpsc/error.rs
@@ -19,7 +19,7 @@
/// This enumeration is the list of the possible error outcomes for the
/// [try_send](super::Sender::try_send) method.
-#[derive(Debug)]
+#[derive(Debug, Eq, PartialEq)]
pub enum TrySendError<T> {
/// The data could not be sent on the channel because the channel is
/// currently full and sending would require blocking.
@@ -78,7 +78,7 @@
// ===== RecvError =====
/// Error returned by `Receiver`.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
#[doc(hidden)]
#[deprecated(note = "This type is unused because recv returns an Option.")]
pub struct RecvError(());
@@ -96,7 +96,7 @@
cfg_time! {
// ===== SendTimeoutError =====
- #[derive(Debug)]
+ #[derive(Debug, Eq, PartialEq)]
/// Error returned by [`Sender::send_timeout`](super::Sender::send_timeout)].
pub enum SendTimeoutError<T> {
/// The data could not be sent on the channel because the channel is
diff --git a/src/sync/mpsc/list.rs b/src/sync/mpsc/list.rs
index e4eeb45..10b2957 100644
--- a/src/sync/mpsc/list.rs
+++ b/src/sync/mpsc/list.rs
@@ -44,7 +44,7 @@
pub(crate) fn channel<T>() -> (Tx<T>, Rx<T>) {
// Create the initial block shared between the tx and rx halves.
- let initial_block = Box::new(Block::new(0));
+ let initial_block = Block::new(0);
let initial_block_ptr = Box::into_raw(initial_block);
let tx = Tx {
diff --git a/src/sync/mpsc/mod.rs b/src/sync/mpsc/mod.rs
index 879e3dc..33889fa 100644
--- a/src/sync/mpsc/mod.rs
+++ b/src/sync/mpsc/mod.rs
@@ -21,6 +21,9 @@
//! when additional capacity is available. In other words, the channel provides
//! backpressure.
//!
+//! This channel is also suitable for the single-producer single-consumer
+//! use-case. (Unless you only need to send one message, in which case you
+//! should use the [oneshot] channel.)
//!
//! # Disconnection
//!
@@ -58,6 +61,22 @@
//! [crossbeam][crossbeam-unbounded]. Similarly, for sending a message _from sync
//! to async_, you should use an unbounded Tokio `mpsc` channel.
//!
+//! Please be aware that the above remarks were written with the `mpsc` channel
+//! in mind, but they can also be generalized to other kinds of channels. In
+//! general, any channel method that isn't marked async can be called anywhere,
+//! including outside of the runtime. For example, sending a message on a
+//! [oneshot] channel from outside the runtime is perfectly fine.
+//!
+//! # Multiple runtimes
+//!
+//! The mpsc channel does not care about which runtime you use it in, and can be
+//! used to send messages from one runtime to another. It can also be used in
+//! non-Tokio runtimes.
+//!
+//! There is one exception to the above: the [`send_timeout`] must be used from
+//! within a Tokio runtime, however it is still not tied to one specific Tokio
+//! runtime, and the sender may be moved from one Tokio runtime to another.
+//!
//! [`Sender`]: crate::sync::mpsc::Sender
//! [`Receiver`]: crate::sync::mpsc::Receiver
//! [bounded-send]: crate::sync::mpsc::Sender::send()
@@ -66,21 +85,25 @@
//! [blocking-recv]: crate::sync::mpsc::Receiver::blocking_recv()
//! [`UnboundedSender`]: crate::sync::mpsc::UnboundedSender
//! [`UnboundedReceiver`]: crate::sync::mpsc::UnboundedReceiver
+//! [oneshot]: crate::sync::oneshot
//! [`Handle::block_on`]: crate::runtime::Handle::block_on()
//! [std-unbounded]: std::sync::mpsc::channel
//! [crossbeam-unbounded]: https://docs.rs/crossbeam/*/crossbeam/channel/fn.unbounded.html
+//! [`send_timeout`]: crate::sync::mpsc::Sender::send_timeout
pub(super) mod block;
mod bounded;
-pub use self::bounded::{channel, OwnedPermit, Permit, Receiver, Sender};
+pub use self::bounded::{channel, OwnedPermit, Permit, Receiver, Sender, WeakSender};
mod chan;
pub(super) mod list;
mod unbounded;
-pub use self::unbounded::{unbounded_channel, UnboundedReceiver, UnboundedSender};
+pub use self::unbounded::{
+ unbounded_channel, UnboundedReceiver, UnboundedSender, WeakUnboundedSender,
+};
pub mod error;
diff --git a/src/sync/mpsc/unbounded.rs b/src/sync/mpsc/unbounded.rs
index b133f9f..5010204 100644
--- a/src/sync/mpsc/unbounded.rs
+++ b/src/sync/mpsc/unbounded.rs
@@ -1,4 +1,4 @@
-use crate::loom::sync::atomic::AtomicUsize;
+use crate::loom::sync::{atomic::AtomicUsize, Arc};
use crate::sync::mpsc::chan;
use crate::sync::mpsc::error::{SendError, TryRecvError};
@@ -13,6 +13,40 @@
chan: chan::Tx<T, Semaphore>,
}
+/// An unbounded sender that does not prevent the channel from being closed.
+///
+/// If all [`UnboundedSender`] instances of a channel were dropped and only
+/// `WeakUnboundedSender` instances remain, the channel is closed.
+///
+/// In order to send messages, the `WeakUnboundedSender` needs to be upgraded using
+/// [`WeakUnboundedSender::upgrade`], which returns `Option<UnboundedSender>`. It returns `None`
+/// if all `UnboundedSender`s have been dropped, and otherwise it returns an `UnboundedSender`.
+///
+/// [`UnboundedSender`]: UnboundedSender
+/// [`WeakUnboundedSender::upgrade`]: WeakUnboundedSender::upgrade
+///
+/// #Examples
+///
+/// ```
+/// use tokio::sync::mpsc::unbounded_channel;
+///
+/// #[tokio::main]
+/// async fn main() {
+/// let (tx, _rx) = unbounded_channel::<i32>();
+/// let tx_weak = tx.downgrade();
+///
+/// // Upgrading will succeed because `tx` still exists.
+/// assert!(tx_weak.upgrade().is_some());
+///
+/// // If we drop `tx`, then it will fail.
+/// drop(tx);
+/// assert!(tx_weak.clone().upgrade().is_none());
+/// }
+/// ```
+pub struct WeakUnboundedSender<T> {
+ chan: Arc<chan::Chan<T, Semaphore>>,
+}
+
impl<T> Clone for UnboundedSender<T> {
fn clone(&self) -> Self {
UnboundedSender {
@@ -61,7 +95,7 @@
/// the channel. Using an `unbounded` channel has the ability of causing the
/// process to run out of memory. In this case, the process will be aborted.
pub fn unbounded_channel<T>() -> (UnboundedSender<T>, UnboundedReceiver<T>) {
- let (tx, rx) = chan::channel(AtomicUsize::new(0));
+ let (tx, rx) = chan::channel(Semaphore(AtomicUsize::new(0)));
let tx = UnboundedSender::new(tx);
let rx = UnboundedReceiver::new(rx);
@@ -70,7 +104,8 @@
}
/// No capacity
-type Semaphore = AtomicUsize;
+#[derive(Debug)]
+pub(crate) struct Semaphore(pub(crate) AtomicUsize);
impl<T> UnboundedReceiver<T> {
pub(crate) fn new(chan: chan::Rx<T, Semaphore>) -> UnboundedReceiver<T> {
@@ -79,8 +114,14 @@
/// Receives the next value for this receiver.
///
- /// `None` is returned when all `Sender` halves have dropped, indicating
- /// that no further values can be sent on the channel.
+ /// This method returns `None` if the channel has been closed and there are
+ /// no remaining messages in the channel's buffer. This indicates that no
+ /// further values can ever be received from this `Receiver`. The channel is
+ /// closed when all senders have been dropped, or when [`close`] is called.
+ ///
+ /// If there are no messages in the channel's buffer, but the channel has
+ /// not yet been closed, this method will sleep until a message is sent or
+ /// the channel is closed.
///
/// # Cancel safety
///
@@ -89,6 +130,8 @@
/// completes first, it is guaranteed that no messages were received on this
/// channel.
///
+ /// [`close`]: Self::close
+ ///
/// # Examples
///
/// ```
@@ -198,6 +241,7 @@
/// sync_code.join().unwrap();
/// }
/// ```
+ #[track_caller]
#[cfg(feature = "sync")]
pub fn blocking_recv(&mut self) -> Option<T> {
crate::future::block_on(self.recv())
@@ -207,6 +251,9 @@
///
/// This prevents any further messages from being sent on the channel while
/// still enabling the receiver to drain messages that are buffered.
+ ///
+ /// To guarantee that no messages are dropped, after calling `close()`,
+ /// `recv()` must be called until `None` is returned.
pub fn close(&mut self) {
self.chan.close();
}
@@ -267,7 +314,7 @@
use std::process;
use std::sync::atomic::Ordering::{AcqRel, Acquire};
- let mut curr = self.chan.semaphore().load(Acquire);
+ let mut curr = self.chan.semaphore().0.load(Acquire);
loop {
if curr & 1 == 1 {
@@ -283,6 +330,7 @@
match self
.chan
.semaphore()
+ .0
.compare_exchange(curr, curr + 2, AcqRel, Acquire)
{
Ok(_) => return true,
@@ -370,4 +418,37 @@
pub fn same_channel(&self, other: &Self) -> bool {
self.chan.same_channel(&other.chan)
}
+
+ /// Converts the `UnboundedSender` to a [`WeakUnboundedSender`] that does not count
+ /// towards RAII semantics, i.e. if all `UnboundedSender` instances of the
+ /// channel were dropped and only `WeakUnboundedSender` instances remain,
+ /// the channel is closed.
+ pub fn downgrade(&self) -> WeakUnboundedSender<T> {
+ WeakUnboundedSender {
+ chan: self.chan.downgrade(),
+ }
+ }
+}
+
+impl<T> Clone for WeakUnboundedSender<T> {
+ fn clone(&self) -> Self {
+ WeakUnboundedSender {
+ chan: self.chan.clone(),
+ }
+ }
+}
+
+impl<T> WeakUnboundedSender<T> {
+ /// Tries to convert a WeakUnboundedSender into an [`UnboundedSender`].
+ /// This will return `Some` if there are other `Sender` instances alive and
+ /// the channel wasn't previously dropped, otherwise `None` is returned.
+ pub fn upgrade(&self) -> Option<UnboundedSender<T>> {
+ chan::Tx::upgrade(self.chan.clone()).map(UnboundedSender::new)
+ }
+}
+
+impl<T> fmt::Debug for WeakUnboundedSender<T> {
+ fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt.debug_struct("WeakUnboundedSender").finish()
+ }
}
diff --git a/src/sync/mutex.rs b/src/sync/mutex.rs
index 4d9f988..024755c 100644
--- a/src/sync/mutex.rs
+++ b/src/sync/mutex.rs
@@ -1,6 +1,8 @@
#![cfg_attr(not(feature = "sync"), allow(unreachable_pub, dead_code))]
use crate::sync::batch_semaphore as semaphore;
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+use crate::util::trace;
use std::cell::UnsafeCell;
use std::error::Error;
@@ -124,6 +126,8 @@
/// [`Send`]: trait@std::marker::Send
/// [`lock`]: method@Mutex::lock
pub struct Mutex<T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
s: semaphore::Semaphore,
c: UnsafeCell<T>,
}
@@ -137,7 +141,10 @@
///
/// The lock is automatically released whenever the guard is dropped, at which
/// point `lock` will succeed yet again.
+#[must_use = "if unused the Mutex will immediately unlock"]
pub struct MutexGuard<'a, T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
lock: &'a Mutex<T>,
}
@@ -157,6 +164,8 @@
///
/// [`Arc`]: std::sync::Arc
pub struct OwnedMutexGuard<T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
lock: Arc<Mutex<T>>,
}
@@ -191,8 +200,8 @@
/// `RwLock::try_read` operation will only fail if the lock is currently held
/// by an exclusive writer.
///
-/// `RwLock::try_write` operation will if lock is held by any reader or by an
-/// exclusive writer.
+/// `RwLock::try_write` operation will only fail if the lock is currently held
+/// by any reader or by an exclusive writer.
///
/// [`Mutex::try_lock`]: Mutex::try_lock
/// [`RwLock::try_read`]: fn@super::RwLock::try_read
@@ -242,13 +251,42 @@
///
/// let lock = Mutex::new(5);
/// ```
+ #[track_caller]
pub fn new(t: T) -> Self
where
T: Sized,
{
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = std::panic::Location::caller();
+
+ tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "Mutex",
+ kind = "Sync",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ )
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let s = resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = false,
+ );
+ semaphore::Semaphore::new(1)
+ });
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ let s = semaphore::Semaphore::new(1);
+
Self {
c: UnsafeCell::new(t),
- s: semaphore::Semaphore::new(1),
+ s,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -270,6 +308,8 @@
Self {
c: UnsafeCell::new(t),
s: semaphore::Semaphore::const_new(1),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span::none(),
}
}
@@ -297,16 +337,50 @@
/// }
/// ```
pub async fn lock(&self) -> MutexGuard<'_, T> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ trace::async_op(
+ || self.acquire(),
+ self.resource_span.clone(),
+ "Mutex::lock",
+ "poll",
+ false,
+ )
+ .await;
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = true,
+ );
+ });
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
self.acquire().await;
- MutexGuard { lock: self }
+
+ MutexGuard {
+ lock: self,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: self.resource_span.clone(),
+ }
}
- /// Blocking lock this mutex. When the lock has been acquired, function returns a
+ /// Blockingly locks this `Mutex`. When the lock has been acquired, function returns a
/// [`MutexGuard`].
///
/// This method is intended for use cases where you
/// need to use this mutex in asynchronous code as well as in synchronous code.
///
+ /// # Panics
+ ///
+ /// This function panics if called within an asynchronous execution context.
+ ///
+ /// - If you find yourself in an asynchronous execution context and needing
+ /// to call some (synchronous) function which performs one of these
+ /// `blocking_` operations, then consider wrapping that call inside
+ /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking]
+ /// (or [`block_in_place()`][crate::task::block_in_place]).
+ ///
/// # Examples
///
/// ```
@@ -316,25 +390,90 @@
/// #[tokio::main]
/// async fn main() {
/// let mutex = Arc::new(Mutex::new(1));
+ /// let lock = mutex.lock().await;
///
/// let mutex1 = Arc::clone(&mutex);
- /// let sync_code = tokio::task::spawn_blocking(move || {
+ /// let blocking_task = tokio::task::spawn_blocking(move || {
+ /// // This shall block until the `lock` is released.
/// let mut n = mutex1.blocking_lock();
/// *n = 2;
/// });
///
- /// sync_code.await.unwrap();
+ /// assert_eq!(*lock, 1);
+ /// // Release the lock.
+ /// drop(lock);
///
- /// let n = mutex.lock().await;
+ /// // Await the completion of the blocking task.
+ /// blocking_task.await.unwrap();
+ ///
+ /// // Assert uncontended.
+ /// let n = mutex.try_lock().unwrap();
/// assert_eq!(*n, 2);
/// }
///
/// ```
+ #[track_caller]
#[cfg(feature = "sync")]
pub fn blocking_lock(&self) -> MutexGuard<'_, T> {
crate::future::block_on(self.lock())
}
+ /// Blockingly locks this `Mutex`. When the lock has been acquired, function returns an
+ /// [`OwnedMutexGuard`].
+ ///
+ /// This method is identical to [`Mutex::blocking_lock`], except that the returned
+ /// guard references the `Mutex` with an [`Arc`] rather than by borrowing
+ /// it. Therefore, the `Mutex` must be wrapped in an `Arc` to call this
+ /// method, and the guard will live for the `'static` lifetime, as it keeps
+ /// the `Mutex` alive by holding an `Arc`.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if called within an asynchronous execution context.
+ ///
+ /// - If you find yourself in an asynchronous execution context and needing
+ /// to call some (synchronous) function which performs one of these
+ /// `blocking_` operations, then consider wrapping that call inside
+ /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking]
+ /// (or [`block_in_place()`][crate::task::block_in_place]).
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use std::sync::Arc;
+ /// use tokio::sync::Mutex;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let mutex = Arc::new(Mutex::new(1));
+ /// let lock = mutex.lock().await;
+ ///
+ /// let mutex1 = Arc::clone(&mutex);
+ /// let blocking_task = tokio::task::spawn_blocking(move || {
+ /// // This shall block until the `lock` is released.
+ /// let mut n = mutex1.blocking_lock_owned();
+ /// *n = 2;
+ /// });
+ ///
+ /// assert_eq!(*lock, 1);
+ /// // Release the lock.
+ /// drop(lock);
+ ///
+ /// // Await the completion of the blocking task.
+ /// blocking_task.await.unwrap();
+ ///
+ /// // Assert uncontended.
+ /// let n = mutex.try_lock().unwrap();
+ /// assert_eq!(*n, 2);
+ /// }
+ ///
+ /// ```
+ #[track_caller]
+ #[cfg(feature = "sync")]
+ pub fn blocking_lock_owned(self: Arc<Self>) -> OwnedMutexGuard<T> {
+ crate::future::block_on(self.lock_owned())
+ }
+
/// Locks this mutex, causing the current task to yield until the lock has
/// been acquired. When the lock has been acquired, this returns an
/// [`OwnedMutexGuard`].
@@ -368,8 +507,35 @@
///
/// [`Arc`]: std::sync::Arc
pub async fn lock_owned(self: Arc<Self>) -> OwnedMutexGuard<T> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ trace::async_op(
+ || self.acquire(),
+ self.resource_span.clone(),
+ "Mutex::lock_owned",
+ "poll",
+ false,
+ )
+ .await;
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = true,
+ );
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
self.acquire().await;
- OwnedMutexGuard { lock: self }
+
+ OwnedMutexGuard {
+ lock: self,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
+ }
}
async fn acquire(&self) {
@@ -399,7 +565,21 @@
/// ```
pub fn try_lock(&self) -> Result<MutexGuard<'_, T>, TryLockError> {
match self.s.try_acquire(1) {
- Ok(_) => Ok(MutexGuard { lock: self }),
+ Ok(_) => {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = true,
+ );
+ });
+
+ Ok(MutexGuard {
+ lock: self,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: self.resource_span.clone(),
+ })
+ }
Err(_) => Err(TryLockError(())),
}
}
@@ -454,7 +634,24 @@
/// # }
pub fn try_lock_owned(self: Arc<Self>) -> Result<OwnedMutexGuard<T>, TryLockError> {
match self.s.try_acquire(1) {
- Ok(_) => Ok(OwnedMutexGuard { lock: self }),
+ Ok(_) => {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = true,
+ );
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+
+ Ok(OwnedMutexGuard {
+ lock: self,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
+ })
+ }
Err(_) => Err(TryLockError(())),
}
}
@@ -626,7 +823,7 @@
/// # async fn main() {
/// # let mutex = Mutex::new(0u32);
/// # let guard = mutex.lock().await;
- /// # unlock_and_relock(guard).await;
+ /// # let _guard = unlock_and_relock(guard).await;
/// # }
/// ```
#[inline]
@@ -637,7 +834,14 @@
impl<T: ?Sized> Drop for MutexGuard<'_, T> {
fn drop(&mut self) {
- self.lock.s.release(1)
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = false,
+ );
+ });
+ self.lock.s.release(1);
}
}
@@ -699,6 +903,13 @@
impl<T: ?Sized> Drop for OwnedMutexGuard<T> {
fn drop(&mut self) {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ locked = false,
+ );
+ });
self.lock.s.release(1)
}
}
diff --git a/src/sync/notify.rs b/src/sync/notify.rs
index c93ce3b..efe16f9 100644
--- a/src/sync/notify.rs
+++ b/src/sync/notify.rs
@@ -13,6 +13,7 @@
use std::cell::UnsafeCell;
use std::future::Future;
use std::marker::PhantomPinned;
+use std::panic::{RefUnwindSafe, UnwindSafe};
use std::pin::Pin;
use std::ptr::NonNull;
use std::sync::atomic::Ordering::SeqCst;
@@ -26,22 +27,23 @@
/// `Notify` itself does not carry any data. Instead, it is to be used to signal
/// another task to perform an operation.
///
-/// `Notify` can be thought of as a [`Semaphore`] starting with 0 permits.
-/// [`notified().await`] waits for a permit to become available, and [`notify_one()`]
-/// sets a permit **if there currently are no available permits**.
+/// A `Notify` can be thought of as a [`Semaphore`] starting with 0 permits. The
+/// [`notified().await`] method waits for a permit to become available, and
+/// [`notify_one()`] sets a permit **if there currently are no available
+/// permits**.
///
/// The synchronization details of `Notify` are similar to
/// [`thread::park`][park] and [`Thread::unpark`][unpark] from std. A [`Notify`]
/// value contains a single permit. [`notified().await`] waits for the permit to
-/// be made available, consumes the permit, and resumes. [`notify_one()`] sets the
-/// permit, waking a pending task if there is one.
+/// be made available, consumes the permit, and resumes. [`notify_one()`] sets
+/// the permit, waking a pending task if there is one.
///
-/// If `notify_one()` is called **before** `notified().await`, then the next call to
-/// `notified().await` will complete immediately, consuming the permit. Any
-/// subsequent calls to `notified().await` will wait for a new permit.
+/// If `notify_one()` is called **before** `notified().await`, then the next
+/// call to `notified().await` will complete immediately, consuming the permit.
+/// Any subsequent calls to `notified().await` will wait for a new permit.
///
-/// If `notify_one()` is called **multiple** times before `notified().await`, only a
-/// **single** permit is stored. The next call to `notified().await` will
+/// If `notify_one()` is called **multiple** times before `notified().await`,
+/// only a **single** permit is stored. The next call to `notified().await` will
/// complete immediately, but the one after will wait for a new permit.
///
/// # Examples
@@ -70,7 +72,11 @@
/// }
/// ```
///
-/// Unbound mpsc channel.
+/// Unbound multi-producer single-consumer (mpsc) channel.
+///
+/// No wakeups can be lost when using this channel because the call to
+/// `notify_one()` will store a permit in the `Notify`, which the following call
+/// to `notified()` will consume.
///
/// ```
/// use tokio::sync::Notify;
@@ -92,6 +98,8 @@
/// self.notify.notify_one();
/// }
///
+/// // This is a single-consumer channel, so several concurrent calls to
+/// // `recv` are not allowed.
/// pub async fn recv(&self) -> T {
/// loop {
/// // Drain values
@@ -106,10 +114,87 @@
/// }
/// ```
///
+/// Unbound multi-producer multi-consumer (mpmc) channel.
+///
+/// The call to [`enable`] is important because otherwise if you have two
+/// calls to `recv` and two calls to `send` in parallel, the following could
+/// happen:
+///
+/// 1. Both calls to `try_recv` return `None`.
+/// 2. Both new elements are added to the vector.
+/// 3. The `notify_one` method is called twice, adding only a single
+/// permit to the `Notify`.
+/// 4. Both calls to `recv` reach the `Notified` future. One of them
+/// consumes the permit, and the other sleeps forever.
+///
+/// By adding the `Notified` futures to the list by calling `enable` before
+/// `try_recv`, the `notify_one` calls in step three would remove the
+/// futures from the list and mark them notified instead of adding a permit
+/// to the `Notify`. This ensures that both futures are woken.
+///
+/// Notice that this failure can only happen if there are two concurrent calls
+/// to `recv`. This is why the mpsc example above does not require a call to
+/// `enable`.
+///
+/// ```
+/// use tokio::sync::Notify;
+///
+/// use std::collections::VecDeque;
+/// use std::sync::Mutex;
+///
+/// struct Channel<T> {
+/// messages: Mutex<VecDeque<T>>,
+/// notify_on_sent: Notify,
+/// }
+///
+/// impl<T> Channel<T> {
+/// pub fn send(&self, msg: T) {
+/// let mut locked_queue = self.messages.lock().unwrap();
+/// locked_queue.push_back(msg);
+/// drop(locked_queue);
+///
+/// // Send a notification to one of the calls currently
+/// // waiting in a call to `recv`.
+/// self.notify_on_sent.notify_one();
+/// }
+///
+/// pub fn try_recv(&self) -> Option<T> {
+/// let mut locked_queue = self.messages.lock().unwrap();
+/// locked_queue.pop_front()
+/// }
+///
+/// pub async fn recv(&self) -> T {
+/// let future = self.notify_on_sent.notified();
+/// tokio::pin!(future);
+///
+/// loop {
+/// // Make sure that no wakeup is lost if we get
+/// // `None` from `try_recv`.
+/// future.as_mut().enable();
+///
+/// if let Some(msg) = self.try_recv() {
+/// return msg;
+/// }
+///
+/// // Wait for a call to `notify_one`.
+/// //
+/// // This uses `.as_mut()` to avoid consuming the future,
+/// // which lets us call `Pin::set` below.
+/// future.as_mut().await;
+///
+/// // Reset the future in case another call to
+/// // `try_recv` got the message before us.
+/// future.set(self.notify_on_sent.notified());
+/// }
+/// }
+/// }
+/// ```
+///
/// [park]: std::thread::park
/// [unpark]: std::thread::Thread::unpark
/// [`notified().await`]: Notify::notified()
/// [`notify_one()`]: Notify::notify_one()
+/// [`enable`]: Notified::enable()
/// [`Semaphore`]: crate::sync::Semaphore
#[derive(Debug)]
pub struct Notify {
@@ -144,7 +229,18 @@
_p: PhantomPinned,
}
-/// Future returned from [`Notify::notified()`]
+generate_addr_of_methods! {
+ impl<> Waiter {
+ unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<Waiter>> {
+ &self.pointers
+ }
+ }
+}
+
+/// Future returned from [`Notify::notified()`].
+///
+/// This future is fused, so once it has completed, any future calls to poll
+/// will immediately return `Poll::Ready`.
#[derive(Debug)]
pub struct Notified<'a> {
/// The `Notify` being received on.
@@ -248,7 +344,16 @@
/// immediately, consuming that permit. Otherwise, `notified().await` waits
/// for a permit to be made available by the next call to `notify_one()`.
///
+ /// The `Notified` future is not guaranteed to receive wakeups from calls to
+ /// `notify_one()` if it has not yet been polled. See the documentation for
+ /// [`Notified::enable()`] for more details.
+ ///
+ /// The `Notified` future is guaranteed to receive wakeups from
+ /// `notify_waiters()` as soon as it has been created, even if it has not
+ /// yet been polled.
+ ///
/// [`notify_one()`]: Notify::notify_one
+ /// [`Notified::enable()`]: Notified::enable
///
/// # Cancel safety
///
@@ -404,7 +509,7 @@
// transition out of WAITING while the lock is held.
let curr = self.state.load(SeqCst);
- if let EMPTY | NOTIFIED = get_state(curr) {
+ if matches!(get_state(curr), EMPTY | NOTIFIED) {
// There are no waiting tasks. All we need to do is increment the
// number of times this method was called.
atomic_inc_num_notify_waiters_calls(&self.state);
@@ -462,6 +567,9 @@
}
}
+impl UnwindSafe for Notify {}
+impl RefUnwindSafe for Notify {}
+
fn notify_locked(waiters: &mut WaitList, state: &AtomicUsize, curr: usize) -> Option<Waker> {
loop {
match get_state(curr) {
@@ -512,6 +620,114 @@
// ===== impl Notified =====
impl Notified<'_> {
+ /// Adds this future to the list of futures that are ready to receive
+ /// wakeups from calls to [`notify_one`].
+ ///
+ /// Polling the future also adds it to the list, so this method should only
+ /// be used if you want to add the future to the list before the first call
+ /// to `poll`. (In fact, this method is equivalent to calling `poll` except
+ /// that no `Waker` is registered.)
+ ///
+ /// This has no effect on notifications sent using [`notify_waiters`], which
+ /// are received as long as they happen after the creation of the `Notified`
+ /// regardless of whether `enable` or `poll` has been called.
+ ///
+ /// This method returns true if the `Notified` is ready. This happens in the
+ /// following situations:
+ ///
+ /// 1. The `notify_waiters` method was called between the creation of the
+ /// `Notified` and the call to this method.
+ /// 2. This is the first call to `enable` or `poll` on this future, and the
+ /// `Notify` was holding a permit from a previous call to `notify_one`.
+ /// The call consumes the permit in that case.
+ /// 3. The future has previously been enabled or polled, and it has since
+ /// then been marked ready by either consuming a permit from the
+ /// `Notify`, or by a call to `notify_one` or `notify_waiters` that
+ /// removed it from the list of futures ready to receive wakeups.
+ ///
+ /// If this method returns true, any future calls to poll on the same future
+ /// will immediately return `Poll::Ready`.
+ ///
+ /// # Examples
+ ///
+ /// Unbound multi-producer multi-consumer (mpmc) channel.
+ ///
+ /// The call to `enable` is important because otherwise if you have two
+ /// calls to `recv` and two calls to `send` in parallel, the following could
+ /// happen:
+ ///
+ /// 1. Both calls to `try_recv` return `None`.
+ /// 2. Both new elements are added to the vector.
+ /// 3. The `notify_one` method is called twice, adding only a single
+ /// permit to the `Notify`.
+ /// 4. Both calls to `recv` reach the `Notified` future. One of them
+ /// consumes the permit, and the other sleeps forever.
+ ///
+ /// By adding the `Notified` futures to the list by calling `enable` before
+ /// `try_recv`, the `notify_one` calls in step three would remove the
+ /// futures from the list and mark them notified instead of adding a permit
+ /// to the `Notify`. This ensures that both futures are woken.
+ ///
+ /// ```
+ /// use tokio::sync::Notify;
+ ///
+ /// use std::collections::VecDeque;
+ /// use std::sync::Mutex;
+ ///
+ /// struct Channel<T> {
+ /// messages: Mutex<VecDeque<T>>,
+ /// notify_on_sent: Notify,
+ /// }
+ ///
+ /// impl<T> Channel<T> {
+ /// pub fn send(&self, msg: T) {
+ /// let mut locked_queue = self.messages.lock().unwrap();
+ /// locked_queue.push_back(msg);
+ /// drop(locked_queue);
+ ///
+ /// // Send a notification to one of the calls currently
+ /// // waiting in a call to `recv`.
+ /// self.notify_on_sent.notify_one();
+ /// }
+ ///
+ /// pub fn try_recv(&self) -> Option<T> {
+ /// let mut locked_queue = self.messages.lock().unwrap();
+ /// locked_queue.pop_front()
+ /// }
+ ///
+ /// pub async fn recv(&self) -> T {
+ /// let future = self.notify_on_sent.notified();
+ /// tokio::pin!(future);
+ ///
+ /// loop {
+ /// // Make sure that no wakeup is lost if we get
+ /// // `None` from `try_recv`.
+ /// future.as_mut().enable();
+ ///
+ /// if let Some(msg) = self.try_recv() {
+ /// return msg;
+ /// }
+ ///
+ /// // Wait for a call to `notify_one`.
+ /// //
+ /// // This uses `.as_mut()` to avoid consuming the future,
+ /// // which lets us call `Pin::set` below.
+ /// future.as_mut().await;
+ ///
+ /// // Reset the future in case another call to
+ /// // `try_recv` got the message before us.
+ /// future.set(self.notify_on_sent.notified());
+ /// }
+ /// }
+ /// }
+ /// ```
+ ///
+ /// [`notify_one`]: Notify::notify_one()
+ /// [`notify_waiters`]: Notify::notify_waiters()
+ pub fn enable(self: Pin<&mut Self>) -> bool {
+ self.poll_notified(None).is_ready()
+ }
+
/// A custom `project` implementation is used in place of `pin-project-lite`
/// as a custom drop implementation is needed.
fn project(self: Pin<&mut Self>) -> (&Notify, &mut State, &UnsafeCell<Waiter>) {
@@ -525,12 +741,8 @@
(me.notify, &mut me.state, &me.waiter)
}
}
-}
-impl Future for Notified<'_> {
- type Output = ();
-
- fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
+ fn poll_notified(self: Pin<&mut Self>, waker: Option<&Waker>) -> Poll<()> {
use State::*;
let (notify, state, waiter) = self.project();
@@ -556,7 +768,7 @@
// Clone the waker before locking, a waker clone can be
// triggering arbitrary code.
- let waker = cx.waker().clone();
+ let waker = waker.cloned();
// Acquire the lock and attempt to transition to the waiting
// state.
@@ -617,9 +829,11 @@
}
}
- // Safety: called while locked.
- unsafe {
- (*waiter.get()).waker = Some(waker);
+ if waker.is_some() {
+ // Safety: called while locked.
+ unsafe {
+ (*waiter.get()).waker = waker;
+ }
}
// Insert the waiter into the linked list
@@ -651,8 +865,14 @@
*state = Done;
} else {
// Update the waker, if necessary.
- if !w.waker.as_ref().unwrap().will_wake(cx.waker()) {
- w.waker = Some(cx.waker().clone());
+ if let Some(waker) = waker {
+ let should_update = match w.waker.as_ref() {
+ Some(current_waker) => !current_waker.will_wake(waker),
+ None => true,
+ };
+ if should_update {
+ w.waker = Some(waker.clone());
+ }
}
return Poll::Pending;
@@ -673,6 +893,14 @@
}
}
+impl Future for Notified<'_> {
+ type Output = ();
+
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
+ self.poll_notified(Some(cx.waker()))
+ }
+}
+
impl Drop for Notified<'_> {
fn drop(&mut self) {
use State::*;
@@ -683,7 +911,7 @@
// This is where we ensure safety. The `Notified` value is being
// dropped, which means we must ensure that the waiter entry is no
// longer stored in the linked list.
- if let Waiting = *state {
+ if matches!(*state, Waiting) {
let mut waiters = notify.waiters.lock();
let mut notify_state = notify.state.load(SeqCst);
@@ -693,11 +921,9 @@
// being the only `LinkedList` available to the type.
unsafe { waiters.remove(NonNull::new_unchecked(waiter.get())) };
- if waiters.is_empty() {
- if let WAITING = get_state(notify_state) {
- notify_state = set_state(notify_state, EMPTY);
- notify.state.store(notify_state, SeqCst);
- }
+ if waiters.is_empty() && get_state(notify_state) == WAITING {
+ notify_state = set_state(notify_state, EMPTY);
+ notify.state.store(notify_state, SeqCst);
}
// See if the node was notified but not received. In this case, if
@@ -706,7 +932,10 @@
//
// Safety: with the entry removed from the linked list, there can be
// no concurrent access to the entry
- if let Some(NotificationType::OneWaiter) = unsafe { (*waiter.get()).notified } {
+ if matches!(
+ unsafe { (*waiter.get()).notified },
+ Some(NotificationType::OneWaiter)
+ ) {
if let Some(waker) = notify_locked(&mut waiters, ¬ify.state, notify_state) {
drop(waiters);
waker.wake();
@@ -731,8 +960,8 @@
ptr
}
- unsafe fn pointers(mut target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
- NonNull::from(&mut target.as_mut().pointers)
+ unsafe fn pointers(target: NonNull<Waiter>) -> NonNull<linked_list::Pointers<Waiter>> {
+ Waiter::addr_of_pointers(target)
}
}
diff --git a/src/sync/once_cell.rs b/src/sync/once_cell.rs
index d31a40e..081efae 100644
--- a/src/sync/once_cell.rs
+++ b/src/sync/once_cell.rs
@@ -106,7 +106,7 @@
if self.initialized_mut() {
unsafe {
self.value
- .with_mut(|ptr| ptr::drop_in_place((&mut *ptr).as_mut_ptr()));
+ .with_mut(|ptr| ptr::drop_in_place((*ptr).as_mut_ptr()));
};
}
}
@@ -416,7 +416,7 @@
/// Errors that can be returned from [`OnceCell::set`].
///
/// [`OnceCell::set`]: crate::sync::OnceCell::set
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq)]
pub enum SetError<T> {
/// The cell was already initialized when [`OnceCell::set`] was called.
///
diff --git a/src/sync/oneshot.rs b/src/sync/oneshot.rs
index 4fb22ec..fcd7a32 100644
--- a/src/sync/oneshot.rs
+++ b/src/sync/oneshot.rs
@@ -9,6 +9,9 @@
//!
//! Each handle can be used on separate tasks.
//!
+//! Since the `send` method is not async, it can be used anywhere. This includes
+//! sending between two runtimes, and using it from non-async code.
+//!
//! # Examples
//!
//! ```
@@ -119,6 +122,8 @@
use crate::loom::cell::UnsafeCell;
use crate::loom::sync::atomic::AtomicUsize;
use crate::loom::sync::Arc;
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+use crate::util::trace;
use std::fmt;
use std::future::Future;
@@ -212,6 +217,8 @@
#[derive(Debug)]
pub struct Sender<T> {
inner: Option<Arc<Inner<T>>>,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
}
/// Receives a value from the associated [`Sender`].
@@ -220,7 +227,15 @@
/// [`channel`](fn@channel) function.
///
/// This channel has no `recv` method because the receiver itself implements the
-/// [`Future`] trait. To receive a value, `.await` the `Receiver` object directly.
+/// [`Future`] trait. To receive a `Result<T, `[`error::RecvError`]`>`, `.await` the `Receiver` object directly.
+///
+/// The `poll` method on the `Future` trait is allowed to spuriously return
+/// `Poll::Pending` even if the message has been sent. If such a spurious
+/// failure happens, then the caller will be woken when the spurious failure has
+/// been resolved so that the caller can attempt to receive the message again.
+/// Note that receiving such a wakeup does not guarantee that the next call will
+/// succeed — it could fail with another spurious failure. (A spurious failure
+/// does not mean that the message is lost. It is just delayed.)
///
/// [`Future`]: trait@std::future::Future
///
@@ -302,6 +317,12 @@
#[derive(Debug)]
pub struct Receiver<T> {
inner: Option<Arc<Inner<T>>>,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ async_op_span: tracing::Span,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ async_op_poll_span: tracing::Span,
}
pub mod error {
@@ -310,11 +331,13 @@
use std::fmt;
/// Error returned by the `Future` implementation for `Receiver`.
- #[derive(Debug, Eq, PartialEq)]
+ ///
+ /// This error is returned by the receiver when the sender is dropped without sending.
+ #[derive(Debug, Eq, PartialEq, Clone)]
pub struct RecvError(pub(super) ());
/// Error returned by the `try_recv` function on `Receiver`.
- #[derive(Debug, Eq, PartialEq)]
+ #[derive(Debug, Eq, PartialEq, Clone)]
pub enum TryRecvError {
/// The send half of the channel has not yet sent a value.
Empty,
@@ -386,21 +409,21 @@
F: FnOnce(&Waker) -> R,
{
self.0.with(|ptr| {
- let waker: *const Waker = (&*ptr).as_ptr();
+ let waker: *const Waker = (*ptr).as_ptr();
f(&*waker)
})
}
unsafe fn drop_task(&self) {
self.0.with_mut(|ptr| {
- let ptr: *mut Waker = (&mut *ptr).as_mut_ptr();
+ let ptr: *mut Waker = (*ptr).as_mut_ptr();
ptr.drop_in_place();
});
}
unsafe fn set_task(&self, cx: &mut Context<'_>) {
self.0.with_mut(|ptr| {
- let ptr: *mut Waker = (&mut *ptr).as_mut_ptr();
+ let ptr: *mut Waker = (*ptr).as_mut_ptr();
ptr.write(cx.waker().clone());
});
}
@@ -439,7 +462,56 @@
/// }
/// }
/// ```
+#[track_caller]
pub fn channel<T>() -> (Sender<T>, Receiver<T>) {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = std::panic::Location::caller();
+
+ let resource_span = tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "Sender|Receiver",
+ kind = "Sync",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ );
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ tx_dropped = false,
+ tx_dropped.op = "override",
+ )
+ });
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ rx_dropped = false,
+ rx_dropped.op = "override",
+ )
+ });
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ value_sent = false,
+ value_sent.op = "override",
+ )
+ });
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ value_received = false,
+ value_received.op = "override",
+ )
+ });
+
+ resource_span
+ };
+
let inner = Arc::new(Inner {
state: AtomicUsize::new(State::new().as_usize()),
value: UnsafeCell::new(None),
@@ -449,8 +521,27 @@
let tx = Sender {
inner: Some(inner.clone()),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: resource_span.clone(),
};
- let rx = Receiver { inner: Some(inner) };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let async_op_span = resource_span
+ .in_scope(|| tracing::trace_span!("runtime.resource.async_op", source = "Receiver::await"));
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let async_op_poll_span =
+ async_op_span.in_scope(|| tracing::trace_span!("runtime.resource.async_op.poll"));
+
+ let rx = Receiver {
+ inner: Some(inner),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ async_op_span,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ async_op_poll_span,
+ };
(tx, rx)
}
@@ -522,6 +613,15 @@
}
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ value_sent = true,
+ value_sent.op = "override",
+ )
+ });
+
Ok(())
}
@@ -595,7 +695,20 @@
pub async fn closed(&mut self) {
use crate::future::poll_fn;
- poll_fn(|cx| self.poll_closed(cx)).await
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let closed = trace::async_op(
+ || poll_fn(|cx| self.poll_closed(cx)),
+ resource_span,
+ "Sender::closed",
+ "poll_closed",
+ false,
+ );
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let closed = poll_fn(|cx| self.poll_closed(cx));
+
+ closed.await
}
/// Returns `true` if the associated [`Receiver`] handle has been dropped.
@@ -674,7 +787,7 @@
/// ```
pub fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll<()> {
// Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
let inner = self.inner.as_ref().unwrap();
@@ -725,6 +838,14 @@
fn drop(&mut self) {
if let Some(inner) = self.inner.as_ref() {
inner.complete();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ tx_dropped = true,
+ tx_dropped.op = "override",
+ )
+ });
}
}
}
@@ -792,6 +913,14 @@
pub fn close(&mut self) {
if let Some(inner) = self.inner.as_ref() {
inner.close();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ rx_dropped = true,
+ rx_dropped.op = "override",
+ )
+ });
}
}
@@ -804,12 +933,16 @@
/// This function is useful to call from outside the context of an
/// asynchronous task.
///
+ /// Note that unlike the `poll` method, the `try_recv` method cannot fail
+ /// spuriously. Any send or close event that happens before this call to
+ /// `try_recv` will be correctly returned to the caller.
+ ///
/// # Return
///
/// - `Ok(T)` if a value is pending in the channel.
/// - `Err(TryRecvError::Empty)` if no value has been sent yet.
/// - `Err(TryRecvError::Closed)` if the sender has dropped without sending
- /// a value.
+ /// a value, or if the message has already been received.
///
/// # Examples
///
@@ -869,7 +1002,17 @@
// `UnsafeCell`. Therefore, it is now safe for us to access the
// cell.
match unsafe { inner.consume_value() } {
- Some(value) => Ok(value),
+ Some(value) => {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ value_received = true,
+ value_received.op = "override",
+ )
+ });
+ Ok(value)
+ }
None => Err(TryRecvError::Closed),
}
} else if state.is_closed() {
@@ -885,12 +1028,51 @@
self.inner = None;
result
}
+
+ /// Blocking receive to call outside of asynchronous contexts.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if called within an asynchronous execution
+ /// context.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use std::thread;
+ /// use tokio::sync::oneshot;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, rx) = oneshot::channel::<u8>();
+ ///
+ /// let sync_code = thread::spawn(move || {
+ /// assert_eq!(Ok(10), rx.blocking_recv());
+ /// });
+ ///
+ /// let _ = tx.send(10);
+ /// sync_code.join().unwrap();
+ /// }
+ /// ```
+ #[track_caller]
+ #[cfg(feature = "sync")]
+ pub fn blocking_recv(self) -> Result<T, RecvError> {
+ crate::future::block_on(self)
+ }
}
impl<T> Drop for Receiver<T> {
fn drop(&mut self) {
if let Some(inner) = self.inner.as_ref() {
inner.close();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ rx_dropped = true,
+ rx_dropped.op = "override",
+ )
+ });
}
}
}
@@ -900,8 +1082,21 @@
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
// If `inner` is `None`, then `poll()` has already completed.
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _res_span = self.resource_span.clone().entered();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _ao_span = self.async_op_span.clone().entered();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _ao_poll_span = self.async_op_poll_span.clone().entered();
+
let ret = if let Some(inner) = self.as_ref().get_ref().inner.as_ref() {
- ready!(inner.poll_recv(cx))?
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let res = ready!(trace_poll_op!("poll_recv", inner.poll_recv(cx)))?;
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ let res = ready!(inner.poll_recv(cx))?;
+
+ res
} else {
panic!("called after complete");
};
@@ -931,7 +1126,7 @@
fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Result<T, RecvError>> {
// Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
// Load the state
let mut state = State::load(&self.state, Acquire);
diff --git a/src/sync/rwlock.rs b/src/sync/rwlock.rs
index 120bc72..0492e4e 100644
--- a/src/sync/rwlock.rs
+++ b/src/sync/rwlock.rs
@@ -1,5 +1,7 @@
use crate::sync::batch_semaphore::{Semaphore, TryAcquireError};
use crate::sync::mutex::TryLockError;
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+use crate::util::trace;
use std::cell::UnsafeCell;
use std::marker;
use std::marker::PhantomData;
@@ -86,6 +88,9 @@
/// [_write-preferring_]: https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock#Priority_policies
#[derive(Debug)]
pub struct RwLock<T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
+
// maximum number of concurrent readers
mr: u32,
@@ -197,14 +202,55 @@
///
/// let lock = RwLock::new(5);
/// ```
+ #[track_caller]
pub fn new(value: T) -> RwLock<T>
where
T: Sized,
{
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = std::panic::Location::caller();
+ let resource_span = tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "RwLock",
+ kind = "Sync",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ );
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ max_readers = MAX_READS,
+ );
+
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ );
+
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 0,
+ );
+ });
+
+ resource_span
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let s = resource_span.in_scope(|| Semaphore::new(MAX_READS as usize));
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ let s = Semaphore::new(MAX_READS as usize);
+
RwLock {
mr: MAX_READS,
c: UnsafeCell::new(value),
- s: Semaphore::new(MAX_READS as usize),
+ s,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -222,6 +268,7 @@
/// # Panics
///
/// Panics if `max_reads` is more than `u32::MAX >> 3`.
+ #[track_caller]
pub fn with_max_readers(value: T, max_reads: u32) -> RwLock<T>
where
T: Sized,
@@ -231,10 +278,52 @@
"a RwLock may not be created with more than {} readers",
MAX_READS
);
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = std::panic::Location::caller();
+
+ let resource_span = tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "RwLock",
+ kind = "Sync",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ );
+
+ resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ max_readers = max_reads,
+ );
+
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ );
+
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 0,
+ );
+ });
+
+ resource_span
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let s = resource_span.in_scope(|| Semaphore::new(max_reads as usize));
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ let s = Semaphore::new(max_reads as usize);
+
RwLock {
mr: max_reads,
c: UnsafeCell::new(value),
- s: Semaphore::new(max_reads as usize),
+ s,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -257,6 +346,8 @@
mr: MAX_READS,
c: UnsafeCell::new(value),
s: Semaphore::const_new(MAX_READS as usize),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span::none(),
}
}
@@ -281,6 +372,8 @@
mr: max_reads,
c: UnsafeCell::new(value),
s: Semaphore::const_new(max_reads as usize),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span::none(),
}
}
@@ -327,21 +420,98 @@
///
/// // Drop the guard after the spawned task finishes.
/// drop(n);
- ///}
+ /// }
/// ```
pub async fn read(&self) -> RwLockReadGuard<'_, T> {
- self.s.acquire(1).await.unwrap_or_else(|_| {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.s.acquire(1),
+ self.resource_span.clone(),
+ "RwLock::read",
+ "poll",
+ false,
+ );
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.s.acquire(1);
+
+ inner.await.unwrap_or_else(|_| {
// The semaphore was closed. but, we never explicitly close it, and we have a
// handle to it through the Arc, which means that this can never happen.
unreachable!()
});
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "add",
+ )
+ });
+
RwLockReadGuard {
s: &self.s,
data: self.c.get(),
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: self.resource_span.clone(),
}
}
+ /// Blockingly locks this `RwLock` with shared read access.
+ ///
+ /// This method is intended for use cases where you
+ /// need to use this rwlock in asynchronous code as well as in synchronous code.
+ ///
+ /// Returns an RAII guard which will drop the read access of this `RwLock` when dropped.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if called within an asynchronous execution context.
+ ///
+ /// - If you find yourself in an asynchronous execution context and needing
+ /// to call some (synchronous) function which performs one of these
+ /// `blocking_` operations, then consider wrapping that call inside
+ /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking]
+ /// (or [`block_in_place()`][crate::task::block_in_place]).
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use std::sync::Arc;
+ /// use tokio::sync::RwLock;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let rwlock = Arc::new(RwLock::new(1));
+ /// let mut write_lock = rwlock.write().await;
+ ///
+ /// let blocking_task = tokio::task::spawn_blocking({
+ /// let rwlock = Arc::clone(&rwlock);
+ /// move || {
+ /// // This shall block until the `write_lock` is released.
+ /// let read_lock = rwlock.blocking_read();
+ /// assert_eq!(*read_lock, 0);
+ /// }
+ /// });
+ ///
+ /// *write_lock -= 1;
+ /// drop(write_lock); // release the lock.
+ ///
+ /// // Await the completion of the blocking task.
+ /// blocking_task.await.unwrap();
+ ///
+ /// // Assert uncontended.
+ /// assert!(rwlock.try_write().is_ok());
+ /// }
+ /// ```
+ #[track_caller]
+ #[cfg(feature = "sync")]
+ pub fn blocking_read(&self) -> RwLockReadGuard<'_, T> {
+ crate::future::block_on(self.read())
+ }
+
/// Locks this `RwLock` with shared read access, causing the current task
/// to yield until the lock has been acquired.
///
@@ -394,15 +564,42 @@
///}
/// ```
pub async fn read_owned(self: Arc<Self>) -> OwnedRwLockReadGuard<T> {
- self.s.acquire(1).await.unwrap_or_else(|_| {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.s.acquire(1),
+ self.resource_span.clone(),
+ "RwLock::read_owned",
+ "poll",
+ false,
+ );
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.s.acquire(1);
+
+ inner.await.unwrap_or_else(|_| {
// The semaphore was closed. but, we never explicitly close it, and we have a
// handle to it through the Arc, which means that this can never happen.
unreachable!()
});
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "add",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+
OwnedRwLockReadGuard {
data: self.c.get(),
lock: ManuallyDrop::new(self),
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -445,10 +642,21 @@
Err(TryAcquireError::Closed) => unreachable!(),
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "add",
+ )
+ });
+
Ok(RwLockReadGuard {
s: &self.s,
data: self.c.get(),
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: self.resource_span.clone(),
})
}
@@ -497,10 +705,24 @@
Err(TryAcquireError::Closed) => unreachable!(),
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "add",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+
Ok(OwnedRwLockReadGuard {
data: self.c.get(),
lock: ManuallyDrop::new(self),
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
@@ -533,19 +755,98 @@
///}
/// ```
pub async fn write(&self) -> RwLockWriteGuard<'_, T> {
- self.s.acquire(self.mr).await.unwrap_or_else(|_| {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.s.acquire(self.mr),
+ self.resource_span.clone(),
+ "RwLock::write",
+ "poll",
+ false,
+ );
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.s.acquire(self.mr);
+
+ inner.await.unwrap_or_else(|_| {
// The semaphore was closed. but, we never explicitly close it, and we have a
// handle to it through the Arc, which means that this can never happen.
unreachable!()
});
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = true,
+ write_locked.op = "override",
+ )
+ });
+
RwLockWriteGuard {
permits_acquired: self.mr,
s: &self.s,
data: self.c.get(),
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: self.resource_span.clone(),
}
}
+ /// Blockingly locks this `RwLock` with exclusive write access.
+ ///
+ /// This method is intended for use cases where you
+ /// need to use this rwlock in asynchronous code as well as in synchronous code.
+ ///
+ /// Returns an RAII guard which will drop the write access of this `RwLock` when dropped.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if called within an asynchronous execution context.
+ ///
+ /// - If you find yourself in an asynchronous execution context and needing
+ /// to call some (synchronous) function which performs one of these
+ /// `blocking_` operations, then consider wrapping that call inside
+ /// [`spawn_blocking()`][crate::runtime::Handle::spawn_blocking]
+ /// (or [`block_in_place()`][crate::task::block_in_place]).
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use std::sync::Arc;
+ /// use tokio::{sync::RwLock};
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let rwlock = Arc::new(RwLock::new(1));
+ /// let read_lock = rwlock.read().await;
+ ///
+ /// let blocking_task = tokio::task::spawn_blocking({
+ /// let rwlock = Arc::clone(&rwlock);
+ /// move || {
+ /// // This shall block until the `read_lock` is released.
+ /// let mut write_lock = rwlock.blocking_write();
+ /// *write_lock = 2;
+ /// }
+ /// });
+ ///
+ /// assert_eq!(*read_lock, 1);
+ /// // Release the last outstanding read lock.
+ /// drop(read_lock);
+ ///
+ /// // Await the completion of the blocking task.
+ /// blocking_task.await.unwrap();
+ ///
+ /// // Assert uncontended.
+ /// let read_lock = rwlock.try_read().unwrap();
+ /// assert_eq!(*read_lock, 2);
+ /// }
+ /// ```
+ #[track_caller]
+ #[cfg(feature = "sync")]
+ pub fn blocking_write(&self) -> RwLockWriteGuard<'_, T> {
+ crate::future::block_on(self.write())
+ }
+
/// Locks this `RwLock` with exclusive write access, causing the current
/// task to yield until the lock has been acquired.
///
@@ -582,16 +883,43 @@
///}
/// ```
pub async fn write_owned(self: Arc<Self>) -> OwnedRwLockWriteGuard<T> {
- self.s.acquire(self.mr).await.unwrap_or_else(|_| {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.s.acquire(self.mr),
+ self.resource_span.clone(),
+ "RwLock::write_owned",
+ "poll",
+ false,
+ );
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.s.acquire(self.mr);
+
+ inner.await.unwrap_or_else(|_| {
// The semaphore was closed. but, we never explicitly close it, and we have a
// handle to it through the Arc, which means that this can never happen.
unreachable!()
});
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = true,
+ write_locked.op = "override",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+
OwnedRwLockWriteGuard {
permits_acquired: self.mr,
data: self.c.get(),
lock: ManuallyDrop::new(self),
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -625,11 +953,22 @@
Err(TryAcquireError::Closed) => unreachable!(),
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = true,
+ write_locked.op = "override",
+ )
+ });
+
Ok(RwLockWriteGuard {
permits_acquired: self.mr,
s: &self.s,
data: self.c.get(),
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: self.resource_span.clone(),
})
}
@@ -670,11 +1009,25 @@
Err(TryAcquireError::Closed) => unreachable!(),
}
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = true,
+ write_locked.op = "override",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+
Ok(OwnedRwLockWriteGuard {
permits_acquired: self.mr,
data: self.c.get(),
lock: ManuallyDrop::new(self),
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
diff --git a/src/sync/rwlock/owned_read_guard.rs b/src/sync/rwlock/owned_read_guard.rs
index 1881295..27b71bd 100644
--- a/src/sync/rwlock/owned_read_guard.rs
+++ b/src/sync/rwlock/owned_read_guard.rs
@@ -15,6 +15,8 @@
/// [`read_owned`]: method@crate::sync::RwLock::read_owned
/// [`RwLock`]: struct@crate::sync::RwLock
pub struct OwnedRwLockReadGuard<T: ?Sized, U: ?Sized = T> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) resource_span: tracing::Span,
// ManuallyDrop allows us to destructure into this field without running the destructor.
pub(super) lock: ManuallyDrop<Arc<RwLock<T>>>,
pub(super) data: *const U,
@@ -56,12 +58,17 @@
{
let data = f(&*this) as *const V;
let lock = unsafe { ManuallyDrop::take(&mut this.lock) };
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
OwnedRwLockReadGuard {
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -105,12 +112,17 @@
None => return Err(this),
};
let lock = unsafe { ManuallyDrop::take(&mut this.lock) };
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
Ok(OwnedRwLockReadGuard {
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
}
@@ -145,5 +157,14 @@
fn drop(&mut self) {
self.lock.s.release(1);
unsafe { ManuallyDrop::drop(&mut self.lock) };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "sub",
+ )
+ });
}
}
diff --git a/src/sync/rwlock/owned_write_guard.rs b/src/sync/rwlock/owned_write_guard.rs
index 0a78d28..dbedab4 100644
--- a/src/sync/rwlock/owned_write_guard.rs
+++ b/src/sync/rwlock/owned_write_guard.rs
@@ -16,6 +16,8 @@
/// [`write_owned`]: method@crate::sync::RwLock::write_owned
/// [`RwLock`]: struct@crate::sync::RwLock
pub struct OwnedRwLockWriteGuard<T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) resource_span: tracing::Span,
pub(super) permits_acquired: u32,
// ManuallyDrop allows us to destructure into this field without running the destructor.
pub(super) lock: ManuallyDrop<Arc<RwLock<T>>>,
@@ -64,13 +66,18 @@
let data = f(&mut *this) as *mut U;
let lock = unsafe { ManuallyDrop::take(&mut this.lock) };
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
OwnedRwLockMappedWriteGuard {
permits_acquired,
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -123,13 +130,19 @@
};
let permits_acquired = this.permits_acquired;
let lock = unsafe { ManuallyDrop::take(&mut this.lock) };
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
+
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
Ok(OwnedRwLockMappedWriteGuard {
permits_acquired,
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
@@ -181,15 +194,39 @@
pub fn downgrade(mut self) -> OwnedRwLockReadGuard<T> {
let lock = unsafe { ManuallyDrop::take(&mut self.lock) };
let data = self.data;
+ let to_release = (self.permits_acquired - 1) as usize;
// Release all but one of the permits held by the write guard
- lock.s.release((self.permits_acquired - 1) as usize);
+ lock.s.release(to_release);
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ write_locked.op = "override",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "add",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(self);
+
OwnedRwLockReadGuard {
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
}
@@ -229,6 +266,14 @@
impl<T: ?Sized> Drop for OwnedRwLockWriteGuard<T> {
fn drop(&mut self) {
self.lock.s.release(self.permits_acquired as usize);
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ write_locked.op = "override",
+ )
+ });
unsafe { ManuallyDrop::drop(&mut self.lock) };
}
}
diff --git a/src/sync/rwlock/owned_write_guard_mapped.rs b/src/sync/rwlock/owned_write_guard_mapped.rs
index d88ee01..55a24d9 100644
--- a/src/sync/rwlock/owned_write_guard_mapped.rs
+++ b/src/sync/rwlock/owned_write_guard_mapped.rs
@@ -15,6 +15,8 @@
/// [mapping]: method@crate::sync::OwnedRwLockWriteGuard::map
/// [`OwnedRwLockWriteGuard`]: struct@crate::sync::OwnedRwLockWriteGuard
pub struct OwnedRwLockMappedWriteGuard<T: ?Sized, U: ?Sized = T> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) resource_span: tracing::Span,
pub(super) permits_acquired: u32,
// ManuallyDrop allows us to destructure into this field without running the destructor.
pub(super) lock: ManuallyDrop<Arc<RwLock<T>>>,
@@ -63,13 +65,18 @@
let data = f(&mut *this) as *mut V;
let lock = unsafe { ManuallyDrop::take(&mut this.lock) };
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
OwnedRwLockMappedWriteGuard {
permits_acquired,
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -120,13 +127,18 @@
};
let lock = unsafe { ManuallyDrop::take(&mut this.lock) };
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
Ok(OwnedRwLockMappedWriteGuard {
permits_acquired,
lock: ManuallyDrop::new(lock),
data,
_p: PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
}
@@ -166,6 +178,14 @@
impl<T: ?Sized, U: ?Sized> Drop for OwnedRwLockMappedWriteGuard<T, U> {
fn drop(&mut self) {
self.lock.s.release(self.permits_acquired as usize);
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ write_locked.op = "override",
+ )
+ });
unsafe { ManuallyDrop::drop(&mut self.lock) };
}
}
diff --git a/src/sync/rwlock/read_guard.rs b/src/sync/rwlock/read_guard.rs
index 090b297..f5fc1d6 100644
--- a/src/sync/rwlock/read_guard.rs
+++ b/src/sync/rwlock/read_guard.rs
@@ -12,7 +12,10 @@
///
/// [`read`]: method@crate::sync::RwLock::read
/// [`RwLock`]: struct@crate::sync::RwLock
+#[must_use = "if unused the RwLock will immediately unlock"]
pub struct RwLockReadGuard<'a, T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) resource_span: tracing::Span,
pub(super) s: &'a Semaphore,
pub(super) data: *const T,
pub(super) marker: marker::PhantomData<&'a T>,
@@ -59,12 +62,17 @@
{
let data = f(&*this) as *const U;
let s = this.s;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
RwLockReadGuard {
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -113,12 +121,17 @@
None => return Err(this),
};
let s = this.s;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
Ok(RwLockReadGuard {
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
}
@@ -152,5 +165,14 @@
impl<'a, T: ?Sized> Drop for RwLockReadGuard<'a, T> {
fn drop(&mut self) {
self.s.release(1);
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "sub",
+ )
+ });
}
}
diff --git a/src/sync/rwlock/write_guard.rs b/src/sync/rwlock/write_guard.rs
index 8c80ee7..cefa183 100644
--- a/src/sync/rwlock/write_guard.rs
+++ b/src/sync/rwlock/write_guard.rs
@@ -14,7 +14,10 @@
///
/// [`write`]: method@crate::sync::RwLock::write
/// [`RwLock`]: struct@crate::sync::RwLock
+#[must_use = "if unused the RwLock will immediately unlock"]
pub struct RwLockWriteGuard<'a, T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) resource_span: tracing::Span,
pub(super) permits_acquired: u32,
pub(super) s: &'a Semaphore,
pub(super) data: *mut T,
@@ -66,6 +69,8 @@
let data = f(&mut *this) as *mut U;
let s = this.s;
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
RwLockMappedWriteGuard {
@@ -73,6 +78,8 @@
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -129,6 +136,8 @@
};
let s = this.s;
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
Ok(RwLockMappedWriteGuard {
@@ -136,6 +145,8 @@
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
@@ -188,15 +199,38 @@
/// [`RwLock`]: struct@crate::sync::RwLock
pub fn downgrade(self) -> RwLockReadGuard<'a, T> {
let RwLockWriteGuard { s, data, .. } = self;
-
+ let to_release = (self.permits_acquired - 1) as usize;
// Release all but one of the permits held by the write guard
- s.release((self.permits_acquired - 1) as usize);
+ s.release(to_release);
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ write_locked.op = "override",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ current_readers = 1,
+ current_readers.op = "add",
+ )
+ });
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(self);
+
RwLockReadGuard {
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
}
@@ -236,5 +270,14 @@
impl<'a, T: ?Sized> Drop for RwLockWriteGuard<'a, T> {
fn drop(&mut self) {
self.s.release(self.permits_acquired as usize);
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ write_locked.op = "override",
+ )
+ });
}
}
diff --git a/src/sync/rwlock/write_guard_mapped.rs b/src/sync/rwlock/write_guard_mapped.rs
index 3cf69de..b5c644a 100644
--- a/src/sync/rwlock/write_guard_mapped.rs
+++ b/src/sync/rwlock/write_guard_mapped.rs
@@ -14,6 +14,8 @@
/// [mapping]: method@crate::sync::RwLockWriteGuard::map
/// [`RwLockWriteGuard`]: struct@crate::sync::RwLockWriteGuard
pub struct RwLockMappedWriteGuard<'a, T: ?Sized> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ pub(super) resource_span: tracing::Span,
pub(super) permits_acquired: u32,
pub(super) s: &'a Semaphore,
pub(super) data: *mut T,
@@ -64,13 +66,18 @@
let data = f(&mut *this) as *mut U;
let s = this.s;
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
RwLockMappedWriteGuard {
permits_acquired,
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -126,13 +133,18 @@
};
let s = this.s;
let permits_acquired = this.permits_acquired;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = this.resource_span.clone();
// NB: Forget to avoid drop impl from being called.
mem::forget(this);
+
Ok(RwLockMappedWriteGuard {
permits_acquired,
s,
data,
marker: marker::PhantomData,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
})
}
}
@@ -172,5 +184,14 @@
impl<'a, T: ?Sized> Drop for RwLockMappedWriteGuard<'a, T> {
fn drop(&mut self) {
self.s.release(self.permits_acquired as usize);
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ self.resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ write_locked = false,
+ write_locked.op = "override",
+ )
+ });
}
}
diff --git a/src/sync/semaphore.rs b/src/sync/semaphore.rs
index 839b523..6e5a1a8 100644
--- a/src/sync/semaphore.rs
+++ b/src/sync/semaphore.rs
@@ -1,5 +1,7 @@
use super::batch_semaphore as ll; // low level implementation
use super::{AcquireError, TryAcquireError};
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+use crate::util::trace;
use std::sync::Arc;
/// Counting semaphore performing asynchronous permit acquisition.
@@ -77,6 +79,8 @@
pub struct Semaphore {
/// The low level semaphore
ll_sem: ll::Semaphore,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
}
/// A permit from the semaphore.
@@ -119,10 +123,41 @@
}
impl Semaphore {
+ /// The maximum number of permits which a semaphore can hold. It is `usize::MAX >>> 3`.
+ ///
+ /// Exceeding this limit typically results in a panic.
+ pub const MAX_PERMITS: usize = super::batch_semaphore::Semaphore::MAX_PERMITS;
+
/// Creates a new semaphore with the initial number of permits.
+ ///
+ /// Panics if `permits` exceeds [`Semaphore::MAX_PERMITS`].
+ #[track_caller]
pub fn new(permits: usize) -> Self {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = std::panic::Location::caller();
+
+ tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "Semaphore",
+ kind = "Sync",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ inherits_child_attrs = true,
+ )
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let ll_sem = resource_span.in_scope(|| ll::Semaphore::new(permits));
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ let ll_sem = ll::Semaphore::new(permits);
+
Self {
- ll_sem: ll::Semaphore::new(permits),
+ ll_sem,
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -139,9 +174,16 @@
#[cfg(all(feature = "parking_lot", not(all(loom, test))))]
#[cfg_attr(docsrs, doc(cfg(feature = "parking_lot")))]
pub const fn const_new(permits: usize) -> Self {
- Self {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ return Self {
ll_sem: ll::Semaphore::const_new(permits),
- }
+ resource_span: tracing::Span::none(),
+ };
+
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ return Self {
+ ll_sem: ll::Semaphore::const_new(permits),
+ };
}
/// Returns the current number of available permits.
@@ -151,7 +193,7 @@
/// Adds `n` new permits to the semaphore.
///
- /// The maximum number of permits is `usize::MAX >> 3`, and this function will panic if the limit is exceeded.
+ /// The maximum number of permits is [`Semaphore::MAX_PERMITS`], and this function will panic if the limit is exceeded.
pub fn add_permits(&self, n: usize) {
self.ll_sem.release(n);
}
@@ -191,7 +233,18 @@
/// [`AcquireError`]: crate::sync::AcquireError
/// [`SemaphorePermit`]: crate::sync::SemaphorePermit
pub async fn acquire(&self) -> Result<SemaphorePermit<'_>, AcquireError> {
- self.ll_sem.acquire(1).await?;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.ll_sem.acquire(1),
+ self.resource_span.clone(),
+ "Semaphore::acquire",
+ "poll",
+ true,
+ );
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.ll_sem.acquire(1);
+
+ inner.await?;
Ok(SemaphorePermit {
sem: self,
permits: 1,
@@ -227,7 +280,19 @@
/// [`AcquireError`]: crate::sync::AcquireError
/// [`SemaphorePermit`]: crate::sync::SemaphorePermit
pub async fn acquire_many(&self, n: u32) -> Result<SemaphorePermit<'_>, AcquireError> {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ trace::async_op(
+ || self.ll_sem.acquire(n),
+ self.resource_span.clone(),
+ "Semaphore::acquire_many",
+ "poll",
+ true,
+ )
+ .await?;
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
self.ll_sem.acquire(n).await?;
+
Ok(SemaphorePermit {
sem: self,
permits: n,
@@ -350,7 +415,18 @@
/// [`AcquireError`]: crate::sync::AcquireError
/// [`OwnedSemaphorePermit`]: crate::sync::OwnedSemaphorePermit
pub async fn acquire_owned(self: Arc<Self>) -> Result<OwnedSemaphorePermit, AcquireError> {
- self.ll_sem.acquire(1).await?;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.ll_sem.acquire(1),
+ self.resource_span.clone(),
+ "Semaphore::acquire_owned",
+ "poll",
+ true,
+ );
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.ll_sem.acquire(1);
+
+ inner.await?;
Ok(OwnedSemaphorePermit {
sem: self,
permits: 1,
@@ -403,7 +479,18 @@
self: Arc<Self>,
n: u32,
) -> Result<OwnedSemaphorePermit, AcquireError> {
- self.ll_sem.acquire(n).await?;
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let inner = trace::async_op(
+ || self.ll_sem.acquire(n),
+ self.resource_span.clone(),
+ "Semaphore::acquire_many_owned",
+ "poll",
+ true,
+ );
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let inner = self.ll_sem.acquire(n);
+
+ inner.await?;
Ok(OwnedSemaphorePermit {
sem: self,
permits: n,
@@ -540,6 +627,25 @@
pub fn forget(mut self) {
self.permits = 0;
}
+
+ /// Merge two [`SemaphorePermit`] instances together, consuming `other`
+ /// without releasing the permits it holds.
+ ///
+ /// Permits held by both `self` and `other` are released when `self` drops.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if permits from different [`Semaphore`] instances
+ /// are merged.
+ #[track_caller]
+ pub fn merge(&mut self, mut other: Self) {
+ assert!(
+ std::ptr::eq(self.sem, other.sem),
+ "merging permits from different semaphore instances"
+ );
+ self.permits += other.permits;
+ other.permits = 0;
+ }
}
impl OwnedSemaphorePermit {
@@ -549,9 +655,28 @@
pub fn forget(mut self) {
self.permits = 0;
}
+
+ /// Merge two [`OwnedSemaphorePermit`] instances together, consuming `other`
+ /// without releasing the permits it holds.
+ ///
+ /// Permits held by both `self` and `other` are released when `self` drops.
+ ///
+ /// # Panics
+ ///
+ /// This function panics if permits from different [`Semaphore`] instances
+ /// are merged.
+ #[track_caller]
+ pub fn merge(&mut self, mut other: Self) {
+ assert!(
+ Arc::ptr_eq(&self.sem, &other.sem),
+ "merging permits from different semaphore instances"
+ );
+ self.permits += other.permits;
+ other.permits = 0;
+ }
}
-impl<'a> Drop for SemaphorePermit<'_> {
+impl Drop for SemaphorePermit<'_> {
fn drop(&mut self) {
self.sem.add_permits(self.permits as usize);
}
diff --git a/src/sync/task/atomic_waker.rs b/src/sync/task/atomic_waker.rs
index e1330fb..13aba35 100644
--- a/src/sync/task/atomic_waker.rs
+++ b/src/sync/task/atomic_waker.rs
@@ -1,7 +1,8 @@
#![cfg_attr(any(loom, not(feature = "sync")), allow(dead_code, unreachable_pub))]
use crate::loom::cell::UnsafeCell;
-use crate::loom::sync::atomic::{self, AtomicUsize};
+use crate::loom::hint;
+use crate::loom::sync::atomic::AtomicUsize;
use std::fmt;
use std::panic::{resume_unwind, AssertUnwindSafe, RefUnwindSafe, UnwindSafe};
@@ -281,9 +282,7 @@
waker.wake();
// This is equivalent to a spin lock, so use a spin hint.
- // TODO: once we bump MSRV to 1.49+, use `hint::spin_loop` instead.
- #[allow(deprecated)]
- atomic::spin_loop_hint();
+ hint::spin_loop();
}
state => {
// In this case, a concurrent thread is holding the
diff --git a/src/sync/tests/atomic_waker.rs b/src/sync/tests/atomic_waker.rs
index b167a5d..8ebfb91 100644
--- a/src/sync/tests/atomic_waker.rs
+++ b/src/sync/tests/atomic_waker.rs
@@ -4,7 +4,7 @@
use std::task::Waker;
trait AssertSend: Send {}
-trait AssertSync: Send {}
+trait AssertSync: Sync {}
impl AssertSend for AtomicWaker {}
impl AssertSync for AtomicWaker {}
@@ -12,6 +12,9 @@
impl AssertSend for Waker {}
impl AssertSync for Waker {}
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+
#[test]
fn basic_usage() {
let mut waker = task::spawn(AtomicWaker::new());
@@ -34,6 +37,7 @@
}
#[test]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn atomic_waker_panic_safe() {
use std::panic;
use std::ptr;
diff --git a/src/sync/tests/notify.rs b/src/sync/tests/notify.rs
index 8c9a573..eb0da8f 100644
--- a/src/sync/tests/notify.rs
+++ b/src/sync/tests/notify.rs
@@ -4,6 +4,9 @@
use std::sync::Arc;
use std::task::{Context, RawWaker, RawWakerVTable, Waker};
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+
#[test]
fn notify_clones_waker_before_lock() {
const VTABLE: &RawWakerVTable = &RawWakerVTable::new(clone_w, wake, wake_by_ref, drop_w);
@@ -42,3 +45,37 @@
// The result doesn't matter, we're just testing that we don't deadlock.
let _ = future.poll(&mut cx);
}
+
+#[test]
+fn notify_simple() {
+ let notify = Notify::new();
+
+ let mut fut1 = tokio_test::task::spawn(notify.notified());
+ assert!(fut1.poll().is_pending());
+
+ let mut fut2 = tokio_test::task::spawn(notify.notified());
+ assert!(fut2.poll().is_pending());
+
+ notify.notify_waiters();
+
+ assert!(fut1.poll().is_ready());
+ assert!(fut2.poll().is_ready());
+}
+
+#[test]
+#[cfg(not(tokio_wasm))]
+fn watch_test() {
+ let rt = crate::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap();
+
+ rt.block_on(async {
+ let (tx, mut rx) = crate::sync::watch::channel(());
+
+ crate::spawn(async move {
+ let _ = tx.send(());
+ });
+
+ let _ = rx.changed().await;
+ });
+}
diff --git a/src/sync/tests/semaphore_batch.rs b/src/sync/tests/semaphore_batch.rs
index 9342cd1..d4e35aa 100644
--- a/src/sync/tests/semaphore_batch.rs
+++ b/src/sync/tests/semaphore_batch.rs
@@ -1,6 +1,9 @@
use crate::sync::batch_semaphore::Semaphore;
use tokio_test::*;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+
#[test]
fn poll_acquire_one_available() {
let s = Semaphore::new(100);
@@ -167,6 +170,7 @@
#[test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn validates_max_permits() {
use std::usize;
Semaphore::new((usize::MAX >> 2) + 1);
diff --git a/src/sync/watch.rs b/src/sync/watch.rs
index 7e45c11..6db46b7 100644
--- a/src/sync/watch.rs
+++ b/src/sync/watch.rs
@@ -9,10 +9,10 @@
//! # Usage
//!
//! [`channel`] returns a [`Sender`] / [`Receiver`] pair. These are the producer
-//! and sender halves of the channel. The channel is created with an initial
+//! and consumer halves of the channel. The channel is created with an initial
//! value. The **latest** value stored in the channel is accessed with
//! [`Receiver::borrow()`]. Awaiting [`Receiver::changed()`] waits for a new
-//! value to sent by the [`Sender`] half.
+//! value to be sent by the [`Sender`] half.
//!
//! # Examples
//!
@@ -20,15 +20,15 @@
//! use tokio::sync::watch;
//!
//! # async fn dox() -> Result<(), Box<dyn std::error::Error>> {
-//! let (tx, mut rx) = watch::channel("hello");
+//! let (tx, mut rx) = watch::channel("hello");
//!
-//! tokio::spawn(async move {
-//! while rx.changed().await.is_ok() {
-//! println!("received = {:?}", *rx.borrow());
-//! }
-//! });
+//! tokio::spawn(async move {
+//! while rx.changed().await.is_ok() {
+//! println!("received = {:?}", *rx.borrow());
+//! }
+//! });
//!
-//! tx.send("world")?;
+//! tx.send("world")?;
//! # Ok(())
//! # }
//! ```
@@ -60,6 +60,7 @@
use crate::loom::sync::{Arc, RwLock, RwLockReadGuard};
use std::mem;
use std::ops;
+use std::panic;
/// Receives values from the associated [`Sender`](struct@Sender).
///
@@ -89,11 +90,80 @@
/// Returns a reference to the inner value.
///
/// Outstanding borrows hold a read lock on the inner value. This means that
-/// long lived borrows could cause the produce half to block. It is recommended
-/// to keep the borrow as short lived as possible.
+/// long-lived borrows could cause the producer half to block. It is recommended
+/// to keep the borrow as short-lived as possible. Additionally, if you are
+/// running in an environment that allows `!Send` futures, you must ensure that
+/// the returned `Ref` type is never held alive across an `.await` point,
+/// otherwise, it can lead to a deadlock.
+///
+/// The priority policy of the lock is dependent on the underlying lock
+/// implementation, and this type does not guarantee that any particular policy
+/// will be used. In particular, a producer which is waiting to acquire the lock
+/// in `send` might or might not block concurrent calls to `borrow`, e.g.:
+///
+/// <details><summary>Potential deadlock example</summary>
+///
+/// ```text
+/// // Task 1 (on thread A) | // Task 2 (on thread B)
+/// let _ref1 = rx.borrow(); |
+/// | // will block
+/// | let _ = tx.send(());
+/// // may deadlock |
+/// let _ref2 = rx.borrow(); |
+/// ```
+/// </details>
#[derive(Debug)]
pub struct Ref<'a, T> {
inner: RwLockReadGuard<'a, T>,
+ has_changed: bool,
+}
+
+impl<'a, T> Ref<'a, T> {
+ /// Indicates if the borrowed value is considered as _changed_ since the last
+ /// time it has been marked as seen.
+ ///
+ /// Unlike [`Receiver::has_changed()`], this method does not fail if the channel is closed.
+ ///
+ /// When borrowed from the [`Sender`] this function will always return `false`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::watch;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx) = watch::channel("hello");
+ ///
+ /// tx.send("goodbye").unwrap();
+ /// // The sender does never consider the value as changed.
+ /// assert!(!tx.borrow().has_changed());
+ ///
+ /// // Drop the sender immediately, just for testing purposes.
+ /// drop(tx);
+ ///
+ /// // Even if the sender has already been dropped...
+ /// assert!(rx.has_changed().is_err());
+ /// // ...the modified value is still readable and detected as changed.
+ /// assert_eq!(*rx.borrow(), "goodbye");
+ /// assert!(rx.borrow().has_changed());
+ ///
+ /// // Read the changed value and mark it as seen.
+ /// {
+ /// let received = rx.borrow_and_update();
+ /// assert_eq!(*received, "goodbye");
+ /// assert!(received.has_changed());
+ /// // Release the read lock when leaving this scope.
+ /// }
+ ///
+ /// // Now the value has already been marked as seen and could
+ /// // never be modified again (after the sender has been dropped).
+ /// assert!(!rx.borrow().has_changed());
+ /// }
+ /// ```
+ pub fn has_changed(&self) -> bool {
+ self.has_changed
+ }
}
#[derive(Debug)]
@@ -137,7 +207,7 @@
impl<T: fmt::Debug> std::error::Error for SendError<T> {}
/// Error produced when receiving a change notification.
- #[derive(Debug)]
+ #[derive(Debug, Clone)]
pub struct RecvError(pub(super) ());
// ===== impl RecvError =====
@@ -281,9 +351,29 @@
/// [`changed`] may return immediately even if you have already seen the
/// value with a call to `borrow`.
///
- /// Outstanding borrows hold a read lock. This means that long lived borrows
- /// could cause the send half to block. It is recommended to keep the borrow
- /// as short lived as possible.
+ /// Outstanding borrows hold a read lock on the inner value. This means that
+ /// long-lived borrows could cause the producer half to block. It is recommended
+ /// to keep the borrow as short-lived as possible. Additionally, if you are
+ /// running in an environment that allows `!Send` futures, you must ensure that
+ /// the returned `Ref` type is never held alive across an `.await` point,
+ /// otherwise, it can lead to a deadlock.
+ ///
+ /// The priority policy of the lock is dependent on the underlying lock
+ /// implementation, and this type does not guarantee that any particular policy
+ /// will be used. In particular, a producer which is waiting to acquire the lock
+ /// in `send` might or might not block concurrent calls to `borrow`, e.g.:
+ ///
+ /// <details><summary>Potential deadlock example</summary>
+ ///
+ /// ```text
+ /// // Task 1 (on thread A) | // Task 2 (on thread B)
+ /// let _ref1 = rx.borrow(); |
+ /// | // will block
+ /// | let _ = tx.send(());
+ /// // may deadlock |
+ /// let _ref2 = rx.borrow(); |
+ /// ```
+ /// </details>
///
/// [`changed`]: Receiver::changed
///
@@ -297,25 +387,101 @@
/// ```
pub fn borrow(&self) -> Ref<'_, T> {
let inner = self.shared.value.read().unwrap();
- Ref { inner }
+
+ // After obtaining a read-lock no concurrent writes could occur
+ // and the loaded version matches that of the borrowed reference.
+ let new_version = self.shared.state.load().version();
+ let has_changed = self.version != new_version;
+
+ Ref { inner, has_changed }
}
- /// Returns a reference to the most recently sent value and mark that value
+ /// Returns a reference to the most recently sent value and marks that value
/// as seen.
///
- /// This method marks the value as seen, so [`changed`] will not return
- /// immediately if the newest value is one previously returned by
- /// `borrow_and_update`.
+ /// This method marks the current value as seen. Subsequent calls to [`changed`]
+ /// will not return immediately until the [`Sender`] has modified the shared
+ /// value again.
///
- /// Outstanding borrows hold a read lock. This means that long lived borrows
- /// could cause the send half to block. It is recommended to keep the borrow
- /// as short lived as possible.
+ /// Outstanding borrows hold a read lock on the inner value. This means that
+ /// long-lived borrows could cause the producer half to block. It is recommended
+ /// to keep the borrow as short-lived as possible. Additionally, if you are
+ /// running in an environment that allows `!Send` futures, you must ensure that
+ /// the returned `Ref` type is never held alive across an `.await` point,
+ /// otherwise, it can lead to a deadlock.
+ ///
+ /// The priority policy of the lock is dependent on the underlying lock
+ /// implementation, and this type does not guarantee that any particular policy
+ /// will be used. In particular, a producer which is waiting to acquire the lock
+ /// in `send` might or might not block concurrent calls to `borrow`, e.g.:
+ ///
+ /// <details><summary>Potential deadlock example</summary>
+ ///
+ /// ```text
+ /// // Task 1 (on thread A) | // Task 2 (on thread B)
+ /// let _ref1 = rx1.borrow_and_update(); |
+ /// | // will block
+ /// | let _ = tx.send(());
+ /// // may deadlock |
+ /// let _ref2 = rx2.borrow_and_update(); |
+ /// ```
+ /// </details>
///
/// [`changed`]: Receiver::changed
pub fn borrow_and_update(&mut self) -> Ref<'_, T> {
let inner = self.shared.value.read().unwrap();
- self.version = self.shared.state.load().version();
- Ref { inner }
+
+ // After obtaining a read-lock no concurrent writes could occur
+ // and the loaded version matches that of the borrowed reference.
+ let new_version = self.shared.state.load().version();
+ let has_changed = self.version != new_version;
+
+ // Mark the shared value as seen by updating the version
+ self.version = new_version;
+
+ Ref { inner, has_changed }
+ }
+
+ /// Checks if this channel contains a message that this receiver has not yet
+ /// seen. The new value is not marked as seen.
+ ///
+ /// Although this method is called `has_changed`, it does not check new
+ /// messages for equality, so this call will return true even if the new
+ /// message is equal to the old message.
+ ///
+ /// Returns an error if the channel has been closed.
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::watch;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let (tx, mut rx) = watch::channel("hello");
+ ///
+ /// tx.send("goodbye").unwrap();
+ ///
+ /// assert!(rx.has_changed().unwrap());
+ /// assert_eq!(*rx.borrow_and_update(), "goodbye");
+ ///
+ /// // The value has been marked as seen
+ /// assert!(!rx.has_changed().unwrap());
+ ///
+ /// drop(tx);
+ /// // The `tx` handle has been dropped
+ /// assert!(rx.has_changed().is_err());
+ /// }
+ /// ```
+ pub fn has_changed(&self) -> Result<bool, error::RecvError> {
+ // Load the version from the state
+ let state = self.shared.state.load();
+ if state.is_closed() {
+ // The sender has dropped.
+ return Err(error::RecvError(()));
+ }
+ let new_version = state.version();
+
+ Ok(self.version != new_version)
}
/// Waits for a change notification, then marks the newest value as seen.
@@ -373,6 +539,22 @@
}
}
+ /// Returns `true` if receivers belong to the same channel.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let (tx, rx) = tokio::sync::watch::channel(true);
+ /// let rx2 = rx.clone();
+ /// assert!(rx.same_channel(&rx2));
+ ///
+ /// let (tx3, rx3) = tokio::sync::watch::channel(true);
+ /// assert!(!rx3.same_channel(&rx2));
+ /// ```
+ pub fn same_channel(&self, other: &Self) -> bool {
+ Arc::ptr_eq(&self.shared, &other.shared)
+ }
+
cfg_process_driver! {
pub(crate) fn try_has_changed(&mut self) -> Option<Result<(), error::RecvError>> {
maybe_changed(&self.shared, &mut self.version)
@@ -425,8 +607,22 @@
impl<T> Sender<T> {
/// Sends a new value via the channel, notifying all receivers.
///
- /// This method fails if the channel has been closed, which happens when
- /// every receiver has been dropped.
+ /// This method fails if the channel is closed, which is the case when
+ /// every receiver has been dropped. It is possible to reopen the channel
+ /// using the [`subscribe`] method. However, when `send` fails, the value
+ /// isn't made available for future receivers (but returned with the
+ /// [`SendError`]).
+ ///
+ /// To always make a new value available for future receivers, even if no
+ /// receiver currently exists, one of the other send methods
+ /// ([`send_if_modified`], [`send_modify`], or [`send_replace`]) can be
+ /// used instead.
+ ///
+ /// [`subscribe`]: Sender::subscribe
+ /// [`SendError`]: error::SendError
+ /// [`send_if_modified`]: Sender::send_if_modified
+ /// [`send_modify`]: Sender::send_modify
+ /// [`send_replace`]: Sender::send_replace
pub fn send(&self, value: T) -> Result<(), error::SendError<T>> {
// This is pretty much only useful as a hint anyway, so synchronization isn't critical.
if 0 == self.receiver_count() {
@@ -437,6 +633,145 @@
Ok(())
}
+ /// Modifies the watched value **unconditionally** in-place,
+ /// notifying all receivers.
+ ///
+ /// This can useful for modifying the watched value, without
+ /// having to allocate a new instance. Additionally, this
+ /// method permits sending values even when there are no receivers.
+ ///
+ /// Prefer to use the more versatile function [`Self::send_if_modified()`]
+ /// if the value is only modified conditionally during the mutable borrow
+ /// to prevent unneeded change notifications for unmodified values.
+ ///
+ /// # Panics
+ ///
+ /// This function panics when the invocation of the `modify` closure panics.
+ /// No receivers are notified when panicking. All changes of the watched
+ /// value applied by the closure before panicking will be visible in
+ /// subsequent calls to `borrow`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::watch;
+ ///
+ /// struct State {
+ /// counter: usize,
+ /// }
+ /// let (state_tx, state_rx) = watch::channel(State { counter: 0 });
+ /// state_tx.send_modify(|state| state.counter += 1);
+ /// assert_eq!(state_rx.borrow().counter, 1);
+ /// ```
+ pub fn send_modify<F>(&self, modify: F)
+ where
+ F: FnOnce(&mut T),
+ {
+ self.send_if_modified(|value| {
+ modify(value);
+ true
+ });
+ }
+
+ /// Modifies the watched value **conditionally** in-place,
+ /// notifying all receivers only if modified.
+ ///
+ /// This can useful for modifying the watched value, without
+ /// having to allocate a new instance. Additionally, this
+ /// method permits sending values even when there are no receivers.
+ ///
+ /// The `modify` closure must return `true` if the value has actually
+ /// been modified during the mutable borrow. It should only return `false`
+ /// if the value is guaranteed to be unmodified despite the mutable
+ /// borrow.
+ ///
+ /// Receivers are only notified if the closure returned `true`. If the
+ /// closure has modified the value but returned `false` this results
+ /// in a *silent modification*, i.e. the modified value will be visible
+ /// in subsequent calls to `borrow`, but receivers will not receive
+ /// a change notification.
+ ///
+ /// Returns the result of the closure, i.e. `true` if the value has
+ /// been modified and `false` otherwise.
+ ///
+ /// # Panics
+ ///
+ /// This function panics when the invocation of the `modify` closure panics.
+ /// No receivers are notified when panicking. All changes of the watched
+ /// value applied by the closure before panicking will be visible in
+ /// subsequent calls to `borrow`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::sync::watch;
+ ///
+ /// struct State {
+ /// counter: usize,
+ /// }
+ /// let (state_tx, mut state_rx) = watch::channel(State { counter: 1 });
+ /// let inc_counter_if_odd = |state: &mut State| {
+ /// if state.counter % 2 == 1 {
+ /// state.counter += 1;
+ /// return true;
+ /// }
+ /// false
+ /// };
+ ///
+ /// assert_eq!(state_rx.borrow().counter, 1);
+ ///
+ /// assert!(!state_rx.has_changed().unwrap());
+ /// assert!(state_tx.send_if_modified(inc_counter_if_odd));
+ /// assert!(state_rx.has_changed().unwrap());
+ /// assert_eq!(state_rx.borrow_and_update().counter, 2);
+ ///
+ /// assert!(!state_rx.has_changed().unwrap());
+ /// assert!(!state_tx.send_if_modified(inc_counter_if_odd));
+ /// assert!(!state_rx.has_changed().unwrap());
+ /// assert_eq!(state_rx.borrow_and_update().counter, 2);
+ /// ```
+ pub fn send_if_modified<F>(&self, modify: F) -> bool
+ where
+ F: FnOnce(&mut T) -> bool,
+ {
+ {
+ // Acquire the write lock and update the value.
+ let mut lock = self.shared.value.write().unwrap();
+
+ // Update the value and catch possible panic inside func.
+ let result = panic::catch_unwind(panic::AssertUnwindSafe(|| modify(&mut lock)));
+ match result {
+ Ok(modified) => {
+ if !modified {
+ // Abort, i.e. don't notify receivers if unmodified
+ return false;
+ }
+ // Continue if modified
+ }
+ Err(panicked) => {
+ // Drop the lock to avoid poisoning it.
+ drop(lock);
+ // Forward the panic to the caller.
+ panic::resume_unwind(panicked);
+ // Unreachable
+ }
+ };
+
+ self.shared.state.increment_version();
+
+ // Release the write lock.
+ //
+ // Incrementing the version counter while holding the lock ensures
+ // that receivers are able to figure out the version number of the
+ // value they are currently looking at.
+ drop(lock);
+ }
+
+ self.shared.notify_rx.notify_waiters();
+
+ true
+ }
+
/// Sends a new value via the channel, notifying all receivers and returning
/// the previous value in the channel.
///
@@ -453,35 +788,21 @@
/// assert_eq!(tx.send_replace(2), 1);
/// assert_eq!(tx.send_replace(3), 2);
/// ```
- pub fn send_replace(&self, value: T) -> T {
- let old = {
- // Acquire the write lock and update the value.
- let mut lock = self.shared.value.write().unwrap();
- let old = mem::replace(&mut *lock, value);
+ pub fn send_replace(&self, mut value: T) -> T {
+ // swap old watched value with the new one
+ self.send_modify(|old| mem::swap(old, &mut value));
- self.shared.state.increment_version();
-
- // Release the write lock.
- //
- // Incrementing the version counter while holding the lock ensures
- // that receivers are able to figure out the version number of the
- // value they are currently looking at.
- drop(lock);
-
- old
- };
-
- // Notify all watchers
- self.shared.notify_rx.notify_waiters();
-
- old
+ value
}
/// Returns a reference to the most recently sent value
///
- /// Outstanding borrows hold a read lock. This means that long lived borrows
- /// could cause the send half to block. It is recommended to keep the borrow
- /// as short lived as possible.
+ /// Outstanding borrows hold a read lock on the inner value. This means that
+ /// long-lived borrows could cause the producer half to block. It is recommended
+ /// to keep the borrow as short-lived as possible. Additionally, if you are
+ /// running in an environment that allows `!Send` futures, you must ensure that
+ /// the returned `Ref` type is never held alive across an `.await` point,
+ /// otherwise, it can lead to a deadlock.
///
/// # Examples
///
@@ -493,7 +814,11 @@
/// ```
pub fn borrow(&self) -> Ref<'_, T> {
let inner = self.shared.value.read().unwrap();
- Ref { inner }
+
+ // The sender/producer always sees the current version
+ let has_changed = false;
+
+ Ref { inner, has_changed }
}
/// Checks if the channel has been closed. This happens when all receivers
diff --git a/src/task/blocking.rs b/src/task/blocking.rs
index 825f25f..46756a9 100644
--- a/src/task/blocking.rs
+++ b/src/task/blocking.rs
@@ -70,11 +70,12 @@
/// This function panics if called from a [`current_thread`] runtime.
///
/// [`current_thread`]: fn@crate::runtime::Builder::new_current_thread
+ #[track_caller]
pub fn block_in_place<F, R>(f: F) -> R
where
F: FnOnce() -> R,
{
- crate::runtime::thread_pool::block_in_place(f)
+ crate::runtime::scheduler::multi_thread::block_in_place(f)
}
}
@@ -102,14 +103,24 @@
/// their own. If you want to spawn an ordinary thread, you should use
/// [`thread::spawn`] instead.
///
- /// Closures spawned using `spawn_blocking` cannot be cancelled. When you shut
- /// down the executor, it will wait indefinitely for all blocking operations to
+ /// Closures spawned using `spawn_blocking` cannot be cancelled abruptly; there
+ /// is no standard low level API to cause a thread to stop running. However,
+ /// a useful pattern is to pass some form of "cancellation token" into
+ /// the thread. This could be an [`AtomicBool`] that the task checks periodically.
+ /// Another approach is to have the thread primarily read or write from a channel,
+ /// and to exit when the channel closes; assuming the other side of the channel is dropped
+ /// when cancellation occurs, this will cause the blocking task thread to exit
+ /// soon after as well.
+ ///
+ /// When you shut down the executor, it will wait indefinitely for all blocking operations to
/// finish. You can use [`shutdown_timeout`] to stop waiting for them after a
/// certain timeout. Be aware that this will still not cancel the tasks — they
- /// are simply allowed to keep running after the method returns.
+ /// are simply allowed to keep running after the method returns. It is possible
+ /// for a blocking task to be cancelled if it has not yet started running, but this
+ /// is not guaranteed.
///
/// Note that if you are using the single threaded runtime, this function will
- /// still spawn additional threads for blocking operations. The basic
+ /// still spawn additional threads for blocking operations. The current-thread
/// scheduler's single thread is only used for asynchronous code.
///
/// # Related APIs and patterns for bridging asynchronous and blocking code
@@ -140,6 +151,7 @@
/// [`thread::spawn`]: fn@std::thread::spawn
/// [`shutdown_timeout`]: fn@crate::runtime::Runtime::shutdown_timeout
/// [bridgesync]: https://tokio.rs/tokio/topics/bridging
+ /// [`AtomicBool`]: struct@std::sync::atomic::AtomicBool
///
/// # Examples
///
@@ -188,7 +200,7 @@
/// worker.await.unwrap();
/// # }
/// ```
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub fn spawn_blocking<F, R>(f: F) -> JoinHandle<R>
where
F: FnOnce() -> R + Send + 'static,
diff --git a/src/task/builder.rs b/src/task/builder.rs
index f991fc6..91c400c 100644
--- a/src/task/builder.rs
+++ b/src/task/builder.rs
@@ -1,9 +1,16 @@
#![allow(unreachable_pub)]
-use crate::{runtime::context, task::JoinHandle};
-use std::future::Future;
+use crate::{
+ runtime::Handle,
+ task::{JoinHandle, LocalSet},
+};
+use std::{future::Future, io};
/// Factory which is used to configure the properties of a new task.
///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
/// Methods can be chained in order to configure it.
///
/// Currently, there is only one configuration option:
@@ -41,11 +48,17 @@
/// .spawn(async move {
/// // Process each socket concurrently.
/// process(socket).await
-/// });
+/// })?;
/// }
/// }
/// ```
+/// [unstable]: crate#unstable-features
+/// [`name`]: Builder::name
+/// [`spawn_local`]: Builder::spawn_local
+/// [`spawn`]: Builder::spawn
+/// [`spawn_blocking`]: Builder::spawn_blocking
#[derive(Default, Debug)]
+#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))]
pub struct Builder<'a> {
name: Option<&'a str>,
}
@@ -61,42 +74,128 @@
Self { name: Some(name) }
}
- /// Spawns a task on the executor.
+ /// Spawns a task with this builder's settings on the current runtime.
+ ///
+ /// # Panics
+ ///
+ /// This method panics if called outside of a Tokio runtime.
///
/// See [`task::spawn`](crate::task::spawn) for
/// more details.
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub fn spawn<Fut>(self, future: Fut) -> JoinHandle<Fut::Output>
+ #[track_caller]
+ pub fn spawn<Fut>(self, future: Fut) -> io::Result<JoinHandle<Fut::Output>>
where
Fut: Future + Send + 'static,
Fut::Output: Send + 'static,
{
- super::spawn::spawn_inner(future, self.name)
+ Ok(super::spawn::spawn_inner(future, self.name))
}
- /// Spawns a task on the current thread.
+ /// Spawn a task with this builder's settings on the provided [runtime
+ /// handle].
///
- /// See [`task::spawn_local`](crate::task::spawn_local)
- /// for more details.
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub fn spawn_local<Fut>(self, future: Fut) -> JoinHandle<Fut::Output>
+ /// See [`Handle::spawn`] for more details.
+ ///
+ /// [runtime handle]: crate::runtime::Handle
+ /// [`Handle::spawn`]: crate::runtime::Handle::spawn
+ #[track_caller]
+ pub fn spawn_on<Fut>(self, future: Fut, handle: &Handle) -> io::Result<JoinHandle<Fut::Output>>
+ where
+ Fut: Future + Send + 'static,
+ Fut::Output: Send + 'static,
+ {
+ Ok(handle.spawn_named(future, self.name))
+ }
+
+ /// Spawns `!Send` a task on the current [`LocalSet`] with this builder's
+ /// settings.
+ ///
+ /// The spawned future will be run on the same thread that called `spawn_local`.
+ /// This may only be called from the context of a [local task set][`LocalSet`].
+ ///
+ /// # Panics
+ ///
+ /// This function panics if called outside of a [local task set][`LocalSet`].
+ ///
+ /// See [`task::spawn_local`] for more details.
+ ///
+ /// [`task::spawn_local`]: crate::task::spawn_local
+ /// [`LocalSet`]: crate::task::LocalSet
+ #[track_caller]
+ pub fn spawn_local<Fut>(self, future: Fut) -> io::Result<JoinHandle<Fut::Output>>
where
Fut: Future + 'static,
Fut::Output: 'static,
{
- super::local::spawn_local_inner(future, self.name)
+ Ok(super::local::spawn_local_inner(future, self.name))
+ }
+
+ /// Spawns `!Send` a task on the provided [`LocalSet`] with this builder's
+ /// settings.
+ ///
+ /// See [`LocalSet::spawn_local`] for more details.
+ ///
+ /// [`LocalSet::spawn_local`]: crate::task::LocalSet::spawn_local
+ /// [`LocalSet`]: crate::task::LocalSet
+ #[track_caller]
+ pub fn spawn_local_on<Fut>(
+ self,
+ future: Fut,
+ local_set: &LocalSet,
+ ) -> io::Result<JoinHandle<Fut::Output>>
+ where
+ Fut: Future + 'static,
+ Fut::Output: 'static,
+ {
+ Ok(local_set.spawn_named(future, self.name))
}
/// Spawns blocking code on the blocking threadpool.
///
+ /// # Panics
+ ///
+ /// This method panics if called outside of a Tokio runtime.
+ ///
/// See [`task::spawn_blocking`](crate::task::spawn_blocking)
/// for more details.
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub fn spawn_blocking<Function, Output>(self, function: Function) -> JoinHandle<Output>
+ #[track_caller]
+ pub fn spawn_blocking<Function, Output>(
+ self,
+ function: Function,
+ ) -> io::Result<JoinHandle<Output>>
where
Function: FnOnce() -> Output + Send + 'static,
Output: Send + 'static,
{
- context::current().spawn_blocking_inner(function, self.name)
+ let handle = Handle::current();
+ self.spawn_blocking_on(function, &handle)
+ }
+
+ /// Spawns blocking code on the provided [runtime handle]'s blocking threadpool.
+ ///
+ /// See [`Handle::spawn_blocking`] for more details.
+ ///
+ /// [runtime handle]: crate::runtime::Handle
+ /// [`Handle::spawn_blocking`]: crate::runtime::Handle::spawn_blocking
+ #[track_caller]
+ pub fn spawn_blocking_on<Function, Output>(
+ self,
+ function: Function,
+ handle: &Handle,
+ ) -> io::Result<JoinHandle<Output>>
+ where
+ Function: FnOnce() -> Output + Send + 'static,
+ Output: Send + 'static,
+ {
+ use crate::runtime::Mandatory;
+ let (join_handle, spawn_result) = handle.inner.blocking_spawner().spawn_blocking_inner(
+ function,
+ Mandatory::NonMandatory,
+ self.name,
+ handle,
+ );
+
+ spawn_result?;
+ Ok(join_handle)
}
}
diff --git a/src/task/consume_budget.rs b/src/task/consume_budget.rs
new file mode 100644
index 0000000..1212cfc
--- /dev/null
+++ b/src/task/consume_budget.rs
@@ -0,0 +1,45 @@
+use std::task::Poll;
+
+/// Consumes a unit of budget and returns the execution back to the Tokio
+/// runtime *if* the task's coop budget was exhausted.
+///
+/// The task will only yield if its entire coop budget has been exhausted.
+/// This function can be used in order to insert optional yield points into long
+/// computations that do not use Tokio resources like sockets or semaphores,
+/// without redundantly yielding to the runtime each time.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// # Examples
+///
+/// Make sure that a function which returns a sum of (potentially lots of)
+/// iterated values is cooperative.
+///
+/// ```
+/// async fn sum_iterator(input: &mut impl std::iter::Iterator<Item=i64>) -> i64 {
+/// let mut sum: i64 = 0;
+/// while let Some(i) = input.next() {
+/// sum += i;
+/// tokio::task::consume_budget().await
+/// }
+/// sum
+/// }
+/// ```
+/// [unstable]: crate#unstable-features
+#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "rt"))))]
+pub async fn consume_budget() {
+ let mut status = Poll::Pending;
+
+ crate::future::poll_fn(move |cx| {
+ if status.is_ready() {
+ return status;
+ }
+ status = crate::runtime::coop::poll_proceed(cx).map(|restore| {
+ restore.made_progress();
+ });
+ status
+ })
+ .await
+}
diff --git a/src/task/join_set.rs b/src/task/join_set.rs
new file mode 100644
index 0000000..e6d8d62
--- /dev/null
+++ b/src/task/join_set.rs
@@ -0,0 +1,522 @@
+//! A collection of tasks spawned on a Tokio runtime.
+//!
+//! This module provides the [`JoinSet`] type, a collection which stores a set
+//! of spawned tasks and allows asynchronously awaiting the output of those
+//! tasks as they complete. See the documentation for the [`JoinSet`] type for
+//! details.
+use std::fmt;
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use crate::runtime::Handle;
+#[cfg(tokio_unstable)]
+use crate::task::Id;
+use crate::task::{AbortHandle, JoinError, JoinHandle, LocalSet};
+use crate::util::IdleNotifiedSet;
+
+/// A collection of tasks spawned on a Tokio runtime.
+///
+/// A `JoinSet` can be used to await the completion of some or all of the tasks
+/// in the set. The set is not ordered, and the tasks will be returned in the
+/// order they complete.
+///
+/// All of the tasks must have the same return type `T`.
+///
+/// When the `JoinSet` is dropped, all tasks in the `JoinSet` are immediately aborted.
+///
+/// # Examples
+///
+/// Spawn multiple tasks and wait for them.
+///
+/// ```
+/// use tokio::task::JoinSet;
+///
+/// #[tokio::main]
+/// async fn main() {
+/// let mut set = JoinSet::new();
+///
+/// for i in 0..10 {
+/// set.spawn(async move { i });
+/// }
+///
+/// let mut seen = [false; 10];
+/// while let Some(res) = set.join_next().await {
+/// let idx = res.unwrap();
+/// seen[idx] = true;
+/// }
+///
+/// for i in 0..10 {
+/// assert!(seen[i]);
+/// }
+/// }
+/// ```
+#[cfg_attr(docsrs, doc(cfg(feature = "rt")))]
+pub struct JoinSet<T> {
+ inner: IdleNotifiedSet<JoinHandle<T>>,
+}
+
+/// A variant of [`task::Builder`] that spawns tasks on a [`JoinSet`] rather
+/// than on the current default runtime.
+///
+/// [`task::Builder`]: crate::task::Builder
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))]
+#[must_use = "builders do nothing unless used to spawn a task"]
+pub struct Builder<'a, T> {
+ joinset: &'a mut JoinSet<T>,
+ builder: super::Builder<'a>,
+}
+
+impl<T> JoinSet<T> {
+ /// Create a new `JoinSet`.
+ pub fn new() -> Self {
+ Self {
+ inner: IdleNotifiedSet::new(),
+ }
+ }
+
+ /// Returns the number of tasks currently in the `JoinSet`.
+ pub fn len(&self) -> usize {
+ self.inner.len()
+ }
+
+ /// Returns whether the `JoinSet` is empty.
+ pub fn is_empty(&self) -> bool {
+ self.inner.is_empty()
+ }
+}
+
+impl<T: 'static> JoinSet<T> {
+ /// Returns a [`Builder`] that can be used to configure a task prior to
+ /// spawning it on this `JoinSet`.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::task::JoinSet;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> std::io::Result<()> {
+ /// let mut set = JoinSet::new();
+ ///
+ /// // Use the builder to configure a task's name before spawning it.
+ /// set.build_task()
+ /// .name("my_task")
+ /// .spawn(async { /* ... */ })?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ #[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))]
+ pub fn build_task(&mut self) -> Builder<'_, T> {
+ Builder {
+ builder: super::Builder::new(),
+ joinset: self,
+ }
+ }
+
+ /// Spawn the provided task on the `JoinSet`, returning an [`AbortHandle`]
+ /// that can be used to remotely cancel the task.
+ ///
+ /// The provided future will start running in the background immediately
+ /// when this method is called, even if you don't await anything on this
+ /// `JoinSet`.
+ ///
+ /// # Panics
+ ///
+ /// This method panics if called outside of a Tokio runtime.
+ ///
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ #[track_caller]
+ pub fn spawn<F>(&mut self, task: F) -> AbortHandle
+ where
+ F: Future<Output = T>,
+ F: Send + 'static,
+ T: Send,
+ {
+ self.insert(crate::spawn(task))
+ }
+
+ /// Spawn the provided task on the provided runtime and store it in this
+ /// `JoinSet` returning an [`AbortHandle`] that can be used to remotely
+ /// cancel the task.
+ ///
+ /// The provided future will start running in the background immediately
+ /// when this method is called, even if you don't await anything on this
+ /// `JoinSet`.
+ ///
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ #[track_caller]
+ pub fn spawn_on<F>(&mut self, task: F, handle: &Handle) -> AbortHandle
+ where
+ F: Future<Output = T>,
+ F: Send + 'static,
+ T: Send,
+ {
+ self.insert(handle.spawn(task))
+ }
+
+ /// Spawn the provided task on the current [`LocalSet`] and store it in this
+ /// `JoinSet`, returning an [`AbortHandle`] that can be used to remotely
+ /// cancel the task.
+ ///
+ /// The provided future will start running in the background immediately
+ /// when this method is called, even if you don't await anything on this
+ /// `JoinSet`.
+ ///
+ /// # Panics
+ ///
+ /// This method panics if it is called outside of a `LocalSet`.
+ ///
+ /// [`LocalSet`]: crate::task::LocalSet
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ #[track_caller]
+ pub fn spawn_local<F>(&mut self, task: F) -> AbortHandle
+ where
+ F: Future<Output = T>,
+ F: 'static,
+ {
+ self.insert(crate::task::spawn_local(task))
+ }
+
+ /// Spawn the provided task on the provided [`LocalSet`] and store it in
+ /// this `JoinSet`, returning an [`AbortHandle`] that can be used to
+ /// remotely cancel the task.
+ ///
+ /// Unlike the [`spawn_local`] method, this method may be used to spawn local
+ /// tasks on a `LocalSet` that is _not_ currently running. The provided
+ /// future will start running whenever the `LocalSet` is next started.
+ ///
+ /// [`LocalSet`]: crate::task::LocalSet
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ /// [`spawn_local`]: Self::spawn_local
+ #[track_caller]
+ pub fn spawn_local_on<F>(&mut self, task: F, local_set: &LocalSet) -> AbortHandle
+ where
+ F: Future<Output = T>,
+ F: 'static,
+ {
+ self.insert(local_set.spawn_local(task))
+ }
+
+ fn insert(&mut self, jh: JoinHandle<T>) -> AbortHandle {
+ let abort = jh.abort_handle();
+ let mut entry = self.inner.insert_idle(jh);
+
+ // Set the waker that is notified when the task completes.
+ entry.with_value_and_context(|jh, ctx| jh.set_join_waker(ctx.waker()));
+ abort
+ }
+
+ /// Waits until one of the tasks in the set completes and returns its output.
+ ///
+ /// Returns `None` if the set is empty.
+ ///
+ /// # Cancel Safety
+ ///
+ /// This method is cancel safe. If `join_next` is used as the event in a `tokio::select!`
+ /// statement and some other branch completes first, it is guaranteed that no tasks were
+ /// removed from this `JoinSet`.
+ pub async fn join_next(&mut self) -> Option<Result<T, JoinError>> {
+ crate::future::poll_fn(|cx| self.poll_join_next(cx)).await
+ }
+
+ /// Waits until one of the tasks in the set completes and returns its
+ /// output, along with the [task ID] of the completed task.
+ ///
+ /// Returns `None` if the set is empty.
+ ///
+ /// When this method returns an error, then the id of the task that failed can be accessed
+ /// using the [`JoinError::id`] method.
+ ///
+ /// # Cancel Safety
+ ///
+ /// This method is cancel safe. If `join_next_with_id` is used as the event in a `tokio::select!`
+ /// statement and some other branch completes first, it is guaranteed that no tasks were
+ /// removed from this `JoinSet`.
+ ///
+ /// [task ID]: crate::task::Id
+ /// [`JoinError::id`]: fn@crate::task::JoinError::id
+ #[cfg(tokio_unstable)]
+ #[cfg_attr(docsrs, doc(cfg(tokio_unstable)))]
+ pub async fn join_next_with_id(&mut self) -> Option<Result<(Id, T), JoinError>> {
+ crate::future::poll_fn(|cx| self.poll_join_next_with_id(cx)).await
+ }
+
+ /// Aborts all tasks and waits for them to finish shutting down.
+ ///
+ /// Calling this method is equivalent to calling [`abort_all`] and then calling [`join_next`] in
+ /// a loop until it returns `None`.
+ ///
+ /// This method ignores any panics in the tasks shutting down. When this call returns, the
+ /// `JoinSet` will be empty.
+ ///
+ /// [`abort_all`]: fn@Self::abort_all
+ /// [`join_next`]: fn@Self::join_next
+ pub async fn shutdown(&mut self) {
+ self.abort_all();
+ while self.join_next().await.is_some() {}
+ }
+
+ /// Aborts all tasks on this `JoinSet`.
+ ///
+ /// This does not remove the tasks from the `JoinSet`. To wait for the tasks to complete
+ /// cancellation, you should call `join_next` in a loop until the `JoinSet` is empty.
+ pub fn abort_all(&mut self) {
+ self.inner.for_each(|jh| jh.abort());
+ }
+
+ /// Removes all tasks from this `JoinSet` without aborting them.
+ ///
+ /// The tasks removed by this call will continue to run in the background even if the `JoinSet`
+ /// is dropped.
+ pub fn detach_all(&mut self) {
+ self.inner.drain(drop);
+ }
+
+ /// Polls for one of the tasks in the set to complete.
+ ///
+ /// If this returns `Poll::Ready(Some(_))`, then the task that completed is removed from the set.
+ ///
+ /// When the method returns `Poll::Pending`, the `Waker` in the provided `Context` is scheduled
+ /// to receive a wakeup when a task in the `JoinSet` completes. Note that on multiple calls to
+ /// `poll_join_next`, only the `Waker` from the `Context` passed to the most recent call is
+ /// scheduled to receive a wakeup.
+ ///
+ /// # Returns
+ ///
+ /// This function returns:
+ ///
+ /// * `Poll::Pending` if the `JoinSet` is not empty but there is no task whose output is
+ /// available right now.
+ /// * `Poll::Ready(Some(Ok(value)))` if one of the tasks in this `JoinSet` has completed.
+ /// The `value` is the return value of one of the tasks that completed.
+ /// * `Poll::Ready(Some(Err(err)))` if one of the tasks in this `JoinSet` has panicked or been
+ /// aborted. The `err` is the `JoinError` from the panicked/aborted task.
+ /// * `Poll::Ready(None)` if the `JoinSet` is empty.
+ ///
+ /// Note that this method may return `Poll::Pending` even if one of the tasks has completed.
+ /// This can happen if the [coop budget] is reached.
+ ///
+ /// [coop budget]: crate::task#cooperative-scheduling
+ fn poll_join_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<T, JoinError>>> {
+ // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to
+ // the `notified` list if the waker is notified in the `poll` call below.
+ let mut entry = match self.inner.pop_notified(cx.waker()) {
+ Some(entry) => entry,
+ None => {
+ if self.is_empty() {
+ return Poll::Ready(None);
+ } else {
+ // The waker was set by `pop_notified`.
+ return Poll::Pending;
+ }
+ }
+ };
+
+ let res = entry.with_value_and_context(|jh, ctx| Pin::new(jh).poll(ctx));
+
+ if let Poll::Ready(res) = res {
+ let _entry = entry.remove();
+ Poll::Ready(Some(res))
+ } else {
+ // A JoinHandle generally won't emit a wakeup without being ready unless
+ // the coop limit has been reached. We yield to the executor in this
+ // case.
+ cx.waker().wake_by_ref();
+ Poll::Pending
+ }
+ }
+
+ /// Polls for one of the tasks in the set to complete.
+ ///
+ /// If this returns `Poll::Ready(Some(_))`, then the task that completed is removed from the set.
+ ///
+ /// When the method returns `Poll::Pending`, the `Waker` in the provided `Context` is scheduled
+ /// to receive a wakeup when a task in the `JoinSet` completes. Note that on multiple calls to
+ /// `poll_join_next`, only the `Waker` from the `Context` passed to the most recent call is
+ /// scheduled to receive a wakeup.
+ ///
+ /// # Returns
+ ///
+ /// This function returns:
+ ///
+ /// * `Poll::Pending` if the `JoinSet` is not empty but there is no task whose output is
+ /// available right now.
+ /// * `Poll::Ready(Some(Ok((id, value))))` if one of the tasks in this `JoinSet` has completed.
+ /// The `value` is the return value of one of the tasks that completed, and
+ /// `id` is the [task ID] of that task.
+ /// * `Poll::Ready(Some(Err(err)))` if one of the tasks in this `JoinSet` has panicked or been
+ /// aborted. The `err` is the `JoinError` from the panicked/aborted task.
+ /// * `Poll::Ready(None)` if the `JoinSet` is empty.
+ ///
+ /// Note that this method may return `Poll::Pending` even if one of the tasks has completed.
+ /// This can happen if the [coop budget] is reached.
+ ///
+ /// [coop budget]: crate::task#cooperative-scheduling
+ /// [task ID]: crate::task::Id
+ #[cfg(tokio_unstable)]
+ fn poll_join_next_with_id(
+ &mut self,
+ cx: &mut Context<'_>,
+ ) -> Poll<Option<Result<(Id, T), JoinError>>> {
+ // The call to `pop_notified` moves the entry to the `idle` list. It is moved back to
+ // the `notified` list if the waker is notified in the `poll` call below.
+ let mut entry = match self.inner.pop_notified(cx.waker()) {
+ Some(entry) => entry,
+ None => {
+ if self.is_empty() {
+ return Poll::Ready(None);
+ } else {
+ // The waker was set by `pop_notified`.
+ return Poll::Pending;
+ }
+ }
+ };
+
+ let res = entry.with_value_and_context(|jh, ctx| Pin::new(jh).poll(ctx));
+
+ if let Poll::Ready(res) = res {
+ let entry = entry.remove();
+ // If the task succeeded, add the task ID to the output. Otherwise, the
+ // `JoinError` will already have the task's ID.
+ Poll::Ready(Some(res.map(|output| (entry.id(), output))))
+ } else {
+ // A JoinHandle generally won't emit a wakeup without being ready unless
+ // the coop limit has been reached. We yield to the executor in this
+ // case.
+ cx.waker().wake_by_ref();
+ Poll::Pending
+ }
+ }
+}
+
+impl<T> Drop for JoinSet<T> {
+ fn drop(&mut self) {
+ self.inner.drain(|join_handle| join_handle.abort());
+ }
+}
+
+impl<T> fmt::Debug for JoinSet<T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("JoinSet").field("len", &self.len()).finish()
+ }
+}
+
+impl<T> Default for JoinSet<T> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+// === impl Builder ===
+
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))]
+impl<'a, T: 'static> Builder<'a, T> {
+ /// Assigns a name to the task which will be spawned.
+ pub fn name(self, name: &'a str) -> Self {
+ let builder = self.builder.name(name);
+ Self { builder, ..self }
+ }
+
+ /// Spawn the provided task with this builder's settings and store it in the
+ /// [`JoinSet`], returning an [`AbortHandle`] that can be used to remotely
+ /// cancel the task.
+ ///
+ /// # Returns
+ ///
+ /// An [`AbortHandle`] that can be used to remotely cancel the task.
+ ///
+ /// # Panics
+ ///
+ /// This method panics if called outside of a Tokio runtime.
+ ///
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ #[track_caller]
+ pub fn spawn<F>(self, future: F) -> std::io::Result<AbortHandle>
+ where
+ F: Future<Output = T>,
+ F: Send + 'static,
+ T: Send,
+ {
+ Ok(self.joinset.insert(self.builder.spawn(future)?))
+ }
+
+ /// Spawn the provided task on the provided [runtime handle] with this
+ /// builder's settings, and store it in the [`JoinSet`].
+ ///
+ /// # Returns
+ ///
+ /// An [`AbortHandle`] that can be used to remotely cancel the task.
+ ///
+ ///
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ /// [runtime handle]: crate::runtime::Handle
+ #[track_caller]
+ pub fn spawn_on<F>(self, future: F, handle: &Handle) -> std::io::Result<AbortHandle>
+ where
+ F: Future<Output = T>,
+ F: Send + 'static,
+ T: Send,
+ {
+ Ok(self.joinset.insert(self.builder.spawn_on(future, handle)?))
+ }
+
+ /// Spawn the provided task on the current [`LocalSet`] with this builder's
+ /// settings, and store it in the [`JoinSet`].
+ ///
+ /// # Returns
+ ///
+ /// An [`AbortHandle`] that can be used to remotely cancel the task.
+ ///
+ /// # Panics
+ ///
+ /// This method panics if it is called outside of a `LocalSet`.
+ ///
+ /// [`LocalSet`]: crate::task::LocalSet
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ #[track_caller]
+ pub fn spawn_local<F>(self, future: F) -> std::io::Result<AbortHandle>
+ where
+ F: Future<Output = T>,
+ F: 'static,
+ {
+ Ok(self.joinset.insert(self.builder.spawn_local(future)?))
+ }
+
+ /// Spawn the provided task on the provided [`LocalSet`] with this builder's
+ /// settings, and store it in the [`JoinSet`].
+ ///
+ /// # Returns
+ ///
+ /// An [`AbortHandle`] that can be used to remotely cancel the task.
+ ///
+ /// [`LocalSet`]: crate::task::LocalSet
+ /// [`AbortHandle`]: crate::task::AbortHandle
+ #[track_caller]
+ pub fn spawn_local_on<F>(self, future: F, local_set: &LocalSet) -> std::io::Result<AbortHandle>
+ where
+ F: Future<Output = T>,
+ F: 'static,
+ {
+ Ok(self
+ .joinset
+ .insert(self.builder.spawn_local_on(future, local_set)?))
+ }
+}
+
+// Manual `Debug` impl so that `Builder` is `Debug` regardless of whether `T` is
+// `Debug`.
+#[cfg(all(tokio_unstable, feature = "tracing"))]
+#[cfg_attr(docsrs, doc(cfg(all(tokio_unstable, feature = "tracing"))))]
+impl<'a, T> fmt::Debug for Builder<'a, T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("join_set::Builder")
+ .field("joinset", &self.joinset)
+ .field("builder", &self.builder)
+ .finish()
+ }
+}
diff --git a/src/task/local.rs b/src/task/local.rs
index 4a5d313..0675faa 100644
--- a/src/task/local.rs
+++ b/src/task/local.rs
@@ -1,8 +1,10 @@
//! Runs `!Send` futures on the current thread.
+use crate::loom::cell::UnsafeCell;
use crate::loom::sync::{Arc, Mutex};
use crate::runtime::task::{self, JoinHandle, LocalOwnedTasks, Task};
+use crate::runtime::{context, ThreadId};
use crate::sync::AtomicWaker;
-use crate::util::VecDequeCell;
+use crate::util::RcCell;
use std::cell::Cell;
use std::collections::VecDeque;
@@ -10,6 +12,7 @@
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
+use std::rc::Rc;
use std::task::Poll;
use pin_project_lite::pin_project;
@@ -159,7 +162,7 @@
/// tokio::task::spawn_local(run_task(new_task));
/// }
/// // If the while loop returns, then all the LocalSpawner
- /// // objects have have been dropped.
+ /// // objects have been dropped.
/// });
///
/// // This will return once all senders are dropped and all
@@ -215,7 +218,7 @@
tick: Cell<u8>,
/// State available from thread-local.
- context: Context,
+ context: Rc<Context>,
/// This type should not be Send.
_not_send: PhantomData<*const ()>,
@@ -224,23 +227,44 @@
/// State available from the thread-local.
struct Context {
- /// Collection of all active tasks spawned onto this executor.
- owned: LocalOwnedTasks<Arc<Shared>>,
-
- /// Local run queue sender and receiver.
- queue: VecDequeCell<task::Notified<Arc<Shared>>>,
-
/// State shared between threads.
shared: Arc<Shared>,
+
+ /// True if a task panicked without being handled and the local set is
+ /// configured to shutdown on unhandled panic.
+ unhandled_panic: Cell<bool>,
}
/// LocalSet state shared between threads.
struct Shared {
+ /// # Safety
+ ///
+ /// This field must *only* be accessed from the thread that owns the
+ /// `LocalSet` (i.e., `Thread::current().id() == owner`).
+ local_state: LocalState,
+
/// Remote run queue sender.
queue: Mutex<Option<VecDeque<task::Notified<Arc<Shared>>>>>,
/// Wake the `LocalSet` task.
waker: AtomicWaker,
+
+ /// How to respond to unhandled task panics.
+ #[cfg(tokio_unstable)]
+ pub(crate) unhandled_panic: crate::runtime::UnhandledPanic,
+}
+
+/// Tracks the `LocalSet` state that must only be accessed from the thread that
+/// created the `LocalSet`.
+struct LocalState {
+ /// The `ThreadId` of the thread that owns the `LocalSet`.
+ owner: ThreadId,
+
+ /// Local run queue sender and receiver.
+ local_queue: UnsafeCell<VecDeque<task::Notified<Arc<Shared>>>>,
+
+ /// Collection of all active tasks spawned onto this executor.
+ owned: LocalOwnedTasks<Arc<Shared>>,
}
pin_project! {
@@ -252,17 +276,30 @@
}
}
-scoped_thread_local!(static CURRENT: Context);
+tokio_thread_local!(static CURRENT: LocalData = const { LocalData {
+ ctx: RcCell::new(),
+} });
+
+struct LocalData {
+ ctx: RcCell<Context>,
+}
cfg_rt! {
- /// Spawns a `!Send` future on the local task set.
+ /// Spawns a `!Send` future on the current [`LocalSet`].
///
- /// The spawned future will be run on the same thread that called `spawn_local.`
- /// This may only be called from the context of a local task set.
+ /// The spawned future will run on the same thread that called `spawn_local`.
+ ///
+ /// The provided future will start running in the background immediately
+ /// when `spawn_local` is called, even if you don't await the returned
+ /// `JoinHandle`.
///
/// # Panics
///
- /// - This function panics if called outside of a local task set.
+ /// This function panics if called outside of a [`LocalSet`].
+ ///
+ /// Note that if [`tokio::spawn`] is used from within a `LocalSet`, the
+ /// resulting new task will _not_ be inside the `LocalSet`, so you must use
+ /// use `spawn_local` if you want to stay within the `LocalSet`.
///
/// # Examples
///
@@ -286,7 +323,10 @@
/// }).await;
/// }
/// ```
- #[cfg_attr(tokio_track_caller, track_caller)]
+ ///
+ /// [`LocalSet`]: struct@crate::task::LocalSet
+ /// [`tokio::spawn`]: fn@crate::task::spawn
+ #[track_caller]
pub fn spawn_local<F>(future: F) -> JoinHandle<F::Output>
where
F: Future + 'static,
@@ -295,23 +335,16 @@
spawn_local_inner(future, None)
}
+
+ #[track_caller]
pub(super) fn spawn_local_inner<F>(future: F, name: Option<&str>) -> JoinHandle<F::Output>
where F: Future + 'static,
F::Output: 'static
{
- let future = crate::util::trace::task(future, "local", name);
- CURRENT.with(|maybe_cx| {
- let cx = maybe_cx
- .expect("`spawn_local` called from outside of a `task::LocalSet`");
-
- let (handle, notified) = cx.owned.bind(future, cx.shared.clone());
-
- if let Some(notified) = notified {
- cx.shared.schedule(notified);
- }
-
- handle
- })
+ match CURRENT.with(|LocalData { ctx, .. }| ctx.get()) {
+ None => panic!("`spawn_local` called from outside of a `task::LocalSet`"),
+ Some(cx) => cx.spawn(future, name)
+ }
}
}
@@ -324,29 +357,72 @@
/// How often it check the remote queue first.
const REMOTE_FIRST_INTERVAL: u8 = 31;
+/// Context guard for LocalSet
+pub struct LocalEnterGuard(Option<Rc<Context>>);
+
+impl Drop for LocalEnterGuard {
+ fn drop(&mut self) {
+ CURRENT.with(|LocalData { ctx, .. }| {
+ ctx.set(self.0.take());
+ })
+ }
+}
+
+impl fmt::Debug for LocalEnterGuard {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("LocalEnterGuard").finish()
+ }
+}
+
impl LocalSet {
/// Returns a new local task set.
pub fn new() -> LocalSet {
+ let owner = context::thread_id().expect("cannot create LocalSet during thread shutdown");
+
LocalSet {
tick: Cell::new(0),
- context: Context {
- owned: LocalOwnedTasks::new(),
- queue: VecDequeCell::with_capacity(INITIAL_CAPACITY),
+ context: Rc::new(Context {
shared: Arc::new(Shared {
+ local_state: LocalState {
+ owner,
+ owned: LocalOwnedTasks::new(),
+ local_queue: UnsafeCell::new(VecDeque::with_capacity(INITIAL_CAPACITY)),
+ },
queue: Mutex::new(Some(VecDeque::with_capacity(INITIAL_CAPACITY))),
waker: AtomicWaker::new(),
+ #[cfg(tokio_unstable)]
+ unhandled_panic: crate::runtime::UnhandledPanic::Ignore,
}),
- },
+ unhandled_panic: Cell::new(false),
+ }),
_not_send: PhantomData,
}
}
+ /// Enters the context of this `LocalSet`.
+ ///
+ /// The [`spawn_local`] method will spawn tasks on the `LocalSet` whose
+ /// context you are inside.
+ ///
+ /// [`spawn_local`]: fn@crate::task::spawn_local
+ pub fn enter(&self) -> LocalEnterGuard {
+ CURRENT.with(|LocalData { ctx, .. }| {
+ let old = ctx.replace(Some(self.context.clone()));
+ LocalEnterGuard(old)
+ })
+ }
+
/// Spawns a `!Send` task onto the local task set.
///
/// This task is guaranteed to be run on the current thread.
///
/// Unlike the free function [`spawn_local`], this method may be used to
- /// spawn local tasks when the task set is _not_ running. For example:
+ /// spawn local tasks when the `LocalSet` is _not_ running. The provided
+ /// future will start running once the `LocalSet` is next started, even if
+ /// you don't await the returned `JoinHandle`.
+ ///
+ /// # Examples
+ ///
/// ```rust
/// use tokio::task;
///
@@ -377,22 +453,13 @@
/// }
/// ```
/// [`spawn_local`]: fn@spawn_local
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub fn spawn_local<F>(&self, future: F) -> JoinHandle<F::Output>
where
F: Future + 'static,
F::Output: 'static,
{
- let future = crate::util::trace::task(future, "local", None);
-
- let (handle, notified) = self.context.owned.bind(future, self.context.shared.clone());
-
- if let Some(notified) = notified {
- self.context.shared.schedule(notified);
- }
-
- self.context.shared.waker.wake();
- handle
+ self.spawn_named(future, None)
}
/// Runs a future to completion on the provided runtime, driving any local
@@ -457,6 +524,7 @@
/// [`Runtime::block_on`]: method@crate::runtime::Runtime::block_on
/// [in-place blocking]: fn@crate::task::block_in_place
/// [`spawn_blocking`]: fn@crate::task::spawn_blocking
+ #[track_caller]
#[cfg(feature = "rt")]
#[cfg_attr(docsrs, doc(cfg(feature = "rt")))]
pub fn block_on<F>(&self, rt: &crate::runtime::Runtime, future: F) -> F::Output
@@ -505,10 +573,36 @@
run_until.await
}
+ pub(in crate::task) fn spawn_named<F>(
+ &self,
+ future: F,
+ name: Option<&str>,
+ ) -> JoinHandle<F::Output>
+ where
+ F: Future + 'static,
+ F::Output: 'static,
+ {
+ let handle = self.context.spawn(future, name);
+
+ // Because a task was spawned from *outside* the `LocalSet`, wake the
+ // `LocalSet` future to execute the new task, if it hasn't been woken.
+ //
+ // Spawning via the free fn `spawn` does not require this, as it can
+ // only be called from *within* a future executing on the `LocalSet` —
+ // in that case, the `LocalSet` must already be awake.
+ self.context.shared.waker.wake();
+ handle
+ }
+
/// Ticks the scheduler, returning whether the local future needs to be
/// notified again.
fn tick(&self) -> bool {
for _ in 0..MAX_TASKS_PER_TICK {
+ // Make sure we didn't hit an unhandled panic
+ if self.context.unhandled_panic.get() {
+ panic!("a spawned task panicked and the LocalSet is configured to shutdown on unhandled panic");
+ }
+
match self.next_task() {
// Run the task
//
@@ -518,7 +612,7 @@
// task initially. Because `LocalSet` itself is `!Send`, and
// `spawn_local` spawns into the `LocalSet` on the current
// thread, the invariant is maintained.
- Some(task) => crate::coop::budget(|| task.run()),
+ Some(task) => crate::runtime::coop::budget(|| task.run()),
// We have fully drained the queue of notified tasks, so the
// local future doesn't need to be notified again — it can wait
// until something else wakes a task in the local set.
@@ -540,9 +634,9 @@
.lock()
.as_mut()
.and_then(|queue| queue.pop_front())
- .or_else(|| self.context.queue.pop_front())
+ .or_else(|| self.pop_local())
} else {
- self.context.queue.pop_front().or_else(|| {
+ self.pop_local().or_else(|| {
self.context
.shared
.queue
@@ -552,11 +646,145 @@
})
};
- task.map(|task| self.context.owned.assert_owner(task))
+ task.map(|task| unsafe {
+ // Safety: because the `LocalSet` itself is `!Send`, we know we are
+ // on the same thread if we have access to the `LocalSet`, and can
+ // therefore access the local run queue.
+ self.context.shared.local_state.assert_owner(task)
+ })
+ }
+
+ fn pop_local(&self) -> Option<task::Notified<Arc<Shared>>> {
+ unsafe {
+ // Safety: because the `LocalSet` itself is `!Send`, we know we are
+ // on the same thread if we have access to the `LocalSet`, and can
+ // therefore access the local run queue.
+ self.context.shared.local_state.task_pop_front()
+ }
}
fn with<T>(&self, f: impl FnOnce() -> T) -> T {
- CURRENT.set(&self.context, f)
+ CURRENT.with(|LocalData { ctx, .. }| {
+ struct Reset<'a> {
+ ctx_ref: &'a RcCell<Context>,
+ val: Option<Rc<Context>>,
+ }
+ impl<'a> Drop for Reset<'a> {
+ fn drop(&mut self) {
+ self.ctx_ref.set(self.val.take());
+ }
+ }
+ let old = ctx.replace(Some(self.context.clone()));
+
+ let _reset = Reset {
+ ctx_ref: ctx,
+ val: old,
+ };
+
+ f()
+ })
+ }
+
+ /// This method is like `with`, but it just calls `f` without setting the thread-local if that
+ /// fails.
+ fn with_if_possible<T>(&self, f: impl FnOnce() -> T) -> T {
+ let mut f = Some(f);
+
+ let res = CURRENT.try_with(|LocalData { ctx, .. }| {
+ struct Reset<'a> {
+ ctx_ref: &'a RcCell<Context>,
+ val: Option<Rc<Context>>,
+ }
+ impl<'a> Drop for Reset<'a> {
+ fn drop(&mut self) {
+ self.ctx_ref.replace(self.val.take());
+ }
+ }
+ let old = ctx.replace(Some(self.context.clone()));
+
+ let _reset = Reset {
+ ctx_ref: ctx,
+ val: old,
+ };
+
+ (f.take().unwrap())()
+ });
+
+ match res {
+ Ok(res) => res,
+ Err(_access_error) => (f.take().unwrap())(),
+ }
+ }
+}
+
+cfg_unstable! {
+ impl LocalSet {
+ /// Configure how the `LocalSet` responds to an unhandled panic on a
+ /// spawned task.
+ ///
+ /// By default, an unhandled panic (i.e. a panic not caught by
+ /// [`std::panic::catch_unwind`]) has no impact on the `LocalSet`'s
+ /// execution. The panic is error value is forwarded to the task's
+ /// [`JoinHandle`] and all other spawned tasks continue running.
+ ///
+ /// The `unhandled_panic` option enables configuring this behavior.
+ ///
+ /// * `UnhandledPanic::Ignore` is the default behavior. Panics on
+ /// spawned tasks have no impact on the `LocalSet`'s execution.
+ /// * `UnhandledPanic::ShutdownRuntime` will force the `LocalSet` to
+ /// shutdown immediately when a spawned task panics even if that
+ /// task's `JoinHandle` has not been dropped. All other spawned tasks
+ /// will immediately terminate and further calls to
+ /// [`LocalSet::block_on`] and [`LocalSet::run_until`] will panic.
+ ///
+ /// # Panics
+ ///
+ /// This method panics if called after the `LocalSet` has started
+ /// running.
+ ///
+ /// # Unstable
+ ///
+ /// This option is currently unstable and its implementation is
+ /// incomplete. The API may change or be removed in the future. See
+ /// tokio-rs/tokio#4516 for more details.
+ ///
+ /// # Examples
+ ///
+ /// The following demonstrates a `LocalSet` configured to shutdown on
+ /// panic. The first spawned task panics and results in the `LocalSet`
+ /// shutting down. The second spawned task never has a chance to
+ /// execute. The call to `run_until` will panic due to the runtime being
+ /// forcibly shutdown.
+ ///
+ /// ```should_panic
+ /// use tokio::runtime::UnhandledPanic;
+ ///
+ /// # #[tokio::main]
+ /// # async fn main() {
+ /// tokio::task::LocalSet::new()
+ /// .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ /// .run_until(async {
+ /// tokio::task::spawn_local(async { panic!("boom"); });
+ /// tokio::task::spawn_local(async {
+ /// // This task never completes
+ /// });
+ ///
+ /// // Do some work, but `run_until` will panic before it completes
+ /// # loop { tokio::task::yield_now().await; }
+ /// })
+ /// .await;
+ /// # }
+ /// ```
+ ///
+ /// [`JoinHandle`]: struct@crate::task::JoinHandle
+ pub fn unhandled_panic(&mut self, behavior: crate::runtime::UnhandledPanic) -> &mut Self {
+ // TODO: This should be set as a builder
+ Rc::get_mut(&mut self.context)
+ .and_then(|ctx| Arc::get_mut(&mut ctx.shared))
+ .expect("Unhandled Panic behavior modified after starting LocalSet")
+ .unhandled_panic = behavior;
+ self
+ }
}
}
@@ -578,7 +806,10 @@
// there are still tasks remaining in the run queue.
cx.waker().wake_by_ref();
Poll::Pending
- } else if self.context.owned.is_empty() {
+
+ // Safety: called from the thread that owns `LocalSet`. Because
+ // `LocalSet` is `!Send`, this is safe.
+ } else if unsafe { self.context.shared.local_state.owned_is_empty() } {
// If the scheduler has no remaining futures, we're done!
Poll::Ready(())
} else {
@@ -598,14 +829,34 @@
impl Drop for LocalSet {
fn drop(&mut self) {
- self.with(|| {
+ self.with_if_possible(|| {
// Shut down all tasks in the LocalOwnedTasks and close it to
// prevent new tasks from ever being added.
- self.context.owned.close_and_shutdown_all();
+ unsafe {
+ // Safety: called from the thread that owns `LocalSet`
+ self.context.shared.local_state.close_and_shutdown_all();
+ }
// We already called shutdown on all tasks above, so there is no
// need to call shutdown.
- for task in self.context.queue.take() {
+
+ // Safety: note that this *intentionally* bypasses the unsafe
+ // `Shared::local_queue()` method. This is in order to avoid the
+ // debug assertion that we are on the thread that owns the
+ // `LocalSet`, because on some systems (e.g. at least some macOS
+ // versions), attempting to get the current thread ID can panic due
+ // to the thread's local data that stores the thread ID being
+ // dropped *before* the `LocalSet`.
+ //
+ // Despite avoiding the assertion here, it is safe for us to access
+ // the local queue in `Drop`, because the `LocalSet` itself is
+ // `!Send`, so we can reasonably guarantee that it will not be
+ // `Drop`ped from another thread.
+ let local_queue = unsafe {
+ // Safety: called from the thread that owns `LocalSet`
+ self.context.shared.local_state.take_local_queue()
+ };
+ for task in local_queue {
drop(task);
}
@@ -616,11 +867,41 @@
drop(task);
}
- assert!(self.context.owned.is_empty());
+ // Safety: called from the thread that owns `LocalSet`
+ assert!(unsafe { self.context.shared.local_state.owned_is_empty() });
});
}
}
+// === impl Context ===
+
+impl Context {
+ #[track_caller]
+ fn spawn<F>(&self, future: F, name: Option<&str>) -> JoinHandle<F::Output>
+ where
+ F: Future + 'static,
+ F::Output: 'static,
+ {
+ let id = crate::runtime::task::Id::next();
+ let future = crate::util::trace::task(future, "local", name, id.as_u64());
+
+ // Safety: called from the thread that owns the `LocalSet`
+ let (handle, notified) = {
+ self.shared.local_state.assert_called_from_owner_thread();
+ self.shared
+ .local_state
+ .owned
+ .bind(future, self.shared.clone(), id)
+ };
+
+ if let Some(notified) = notified {
+ self.shared.schedule(notified);
+ }
+
+ handle
+ }
+}
+
// === impl LocalFuture ===
impl<T: Future> Future for RunUntil<'_, T> {
@@ -636,10 +917,10 @@
.waker
.register_by_ref(cx.waker());
- let _no_blocking = crate::runtime::enter::disallow_blocking();
+ let _no_blocking = crate::runtime::context::disallow_block_in_place();
let f = me.future;
- if let Poll::Ready(output) = crate::coop::budget(|| f.poll(cx)) {
+ if let Poll::Ready(output) = f.poll(cx) {
return Poll::Ready(output);
}
@@ -657,21 +938,41 @@
impl Shared {
/// Schedule the provided task on the scheduler.
fn schedule(&self, task: task::Notified<Arc<Self>>) {
- CURRENT.with(|maybe_cx| match maybe_cx {
- Some(cx) if cx.shared.ptr_eq(self) => {
- cx.queue.push_back(task);
- }
- _ => {
- // First check whether the queue is still there (if not, the
- // LocalSet is dropped). Then push to it if so, and if not,
- // do nothing.
- let mut lock = self.queue.lock();
+ CURRENT.with(|localdata| {
+ match localdata.ctx.get() {
+ Some(cx) if cx.shared.ptr_eq(self) => unsafe {
+ // Safety: if the current `LocalSet` context points to this
+ // `LocalSet`, then we are on the thread that owns it.
+ cx.shared.local_state.task_push_back(task);
+ },
- if let Some(queue) = lock.as_mut() {
- queue.push_back(task);
- drop(lock);
+ // We are on the thread that owns the `LocalSet`, so we can
+ // wake to the local queue.
+ _ if context::thread_id().ok() == Some(self.local_state.owner) => {
+ unsafe {
+ // Safety: we just checked that the thread ID matches
+ // the localset's owner, so this is safe.
+ self.local_state.task_push_back(task);
+ }
+ // We still have to wake the `LocalSet`, because it isn't
+ // currently being polled.
self.waker.wake();
}
+
+ // We are *not* on the thread that owns the `LocalSet`, so we
+ // have to wake to the remote queue.
+ _ => {
+ // First, check whether the queue is still there (if not, the
+ // LocalSet is dropped). Then push to it if so, and if not,
+ // do nothing.
+ let mut lock = self.queue.lock();
+
+ if let Some(queue) = lock.as_mut() {
+ queue.push_back(task);
+ drop(lock);
+ self.waker.wake();
+ }
+ }
}
});
}
@@ -681,16 +982,194 @@
}
}
+// This is safe because (and only because) we *pinky pwomise* to never touch the
+// local run queue except from the thread that owns the `LocalSet`.
+unsafe impl Sync for Shared {}
+
impl task::Schedule for Arc<Shared> {
fn release(&self, task: &Task<Self>) -> Option<Task<Self>> {
- CURRENT.with(|maybe_cx| {
- let cx = maybe_cx.expect("scheduler context missing");
- assert!(cx.shared.ptr_eq(self));
- cx.owned.remove(task)
- })
+ // Safety, this is always called from the thread that owns `LocalSet`
+ unsafe { self.local_state.task_remove(task) }
}
fn schedule(&self, task: task::Notified<Self>) {
Shared::schedule(self, task);
}
+
+ cfg_unstable! {
+ fn unhandled_panic(&self) {
+ use crate::runtime::UnhandledPanic;
+
+ match self.unhandled_panic {
+ UnhandledPanic::Ignore => {
+ // Do nothing
+ }
+ UnhandledPanic::ShutdownRuntime => {
+ // This hook is only called from within the runtime, so
+ // `CURRENT` should match with `&self`, i.e. there is no
+ // opportunity for a nested scheduler to be called.
+ CURRENT.with(|LocalData { ctx, .. }| match ctx.get() {
+ Some(cx) if Arc::ptr_eq(self, &cx.shared) => {
+ cx.unhandled_panic.set(true);
+ // Safety: this is always called from the thread that owns `LocalSet`
+ unsafe { cx.shared.local_state.close_and_shutdown_all(); }
+ }
+ _ => unreachable!("runtime core not set in CURRENT thread-local"),
+ })
+ }
+ }
+ }
+ }
+}
+
+impl LocalState {
+ unsafe fn task_pop_front(&self) -> Option<task::Notified<Arc<Shared>>> {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.local_queue.with_mut(|ptr| (*ptr).pop_front())
+ }
+
+ unsafe fn task_push_back(&self, task: task::Notified<Arc<Shared>>) {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.local_queue.with_mut(|ptr| (*ptr).push_back(task))
+ }
+
+ unsafe fn take_local_queue(&self) -> VecDeque<task::Notified<Arc<Shared>>> {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.local_queue.with_mut(|ptr| std::mem::take(&mut (*ptr)))
+ }
+
+ unsafe fn task_remove(&self, task: &Task<Arc<Shared>>) -> Option<Task<Arc<Shared>>> {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.owned.remove(task)
+ }
+
+ /// Returns true if the `LocalSet` does not have any spawned tasks
+ unsafe fn owned_is_empty(&self) -> bool {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.owned.is_empty()
+ }
+
+ unsafe fn assert_owner(
+ &self,
+ task: task::Notified<Arc<Shared>>,
+ ) -> task::LocalNotified<Arc<Shared>> {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.owned.assert_owner(task)
+ }
+
+ unsafe fn close_and_shutdown_all(&self) {
+ // The caller ensures it is called from the same thread that owns
+ // the LocalSet.
+ self.assert_called_from_owner_thread();
+
+ self.owned.close_and_shutdown_all()
+ }
+
+ #[track_caller]
+ fn assert_called_from_owner_thread(&self) {
+ // FreeBSD has some weirdness around thread-local destruction.
+ // TODO: remove this hack when thread id is cleaned up
+ #[cfg(not(any(target_os = "openbsd", target_os = "freebsd")))]
+ debug_assert!(
+ // if we couldn't get the thread ID because we're dropping the local
+ // data, skip the assertion --- the `Drop` impl is not going to be
+ // called from another thread, because `LocalSet` is `!Send`
+ context::thread_id()
+ .map(|id| id == self.owner)
+ .unwrap_or(true),
+ "`LocalSet`'s local run queue must not be accessed by another thread!"
+ );
+ }
+}
+
+// This is `Send` because it is stored in `Shared`. It is up to the caller to
+// ensure they are on the same thread that owns the `LocalSet`.
+unsafe impl Send for LocalState {}
+
+#[cfg(all(test, not(loom)))]
+mod tests {
+ use super::*;
+
+ // Does a `LocalSet` running on a current-thread runtime...basically work?
+ //
+ // This duplicates a test in `tests/task_local_set.rs`, but because this is
+ // a lib test, it wil run under Miri, so this is necessary to catch stacked
+ // borrows violations in the `LocalSet` implementation.
+ #[test]
+ fn local_current_thread_scheduler() {
+ let f = async {
+ LocalSet::new()
+ .run_until(async {
+ spawn_local(async {}).await.unwrap();
+ })
+ .await;
+ };
+ crate::runtime::Builder::new_current_thread()
+ .build()
+ .expect("rt")
+ .block_on(f)
+ }
+
+ // Tests that when a task on a `LocalSet` is woken by an io driver on the
+ // same thread, the task is woken to the localset's local queue rather than
+ // its remote queue.
+ //
+ // This test has to be defined in the `local.rs` file as a lib test, rather
+ // than in `tests/`, because it makes assertions about the local set's
+ // internal state.
+ #[test]
+ fn wakes_to_local_queue() {
+ use super::*;
+ use crate::sync::Notify;
+ let rt = crate::runtime::Builder::new_current_thread()
+ .build()
+ .expect("rt");
+ rt.block_on(async {
+ let local = LocalSet::new();
+ let notify = Arc::new(Notify::new());
+ let task = local.spawn_local({
+ let notify = notify.clone();
+ async move {
+ notify.notified().await;
+ }
+ });
+ let mut run_until = Box::pin(local.run_until(async move {
+ task.await.unwrap();
+ }));
+
+ // poll the run until future once
+ crate::future::poll_fn(|cx| {
+ let _ = run_until.as_mut().poll(cx);
+ Poll::Ready(())
+ })
+ .await;
+
+ notify.notify_one();
+ let task = unsafe { local.context.shared.local_state.task_pop_front() };
+ // TODO(eliza): it would be nice to be able to assert that this is
+ // the local task.
+ assert!(
+ task.is_some(),
+ "task should have been notified to the LocalSet's local queue"
+ );
+ })
+ }
}
diff --git a/src/task/mod.rs b/src/task/mod.rs
index ea98787..9b75370 100644
--- a/src/task/mod.rs
+++ b/src/task/mod.rs
@@ -243,7 +243,7 @@
//!
//! #### unconstrained
//!
-//! If necessary, [`task::unconstrained`] lets you opt out a future of Tokio's cooperative
+//! If necessary, [`task::unconstrained`] lets you opt a future out of of Tokio's cooperative
//! scheduling. When a future is wrapped with `unconstrained`, it will never be forced to yield to
//! Tokio. For example:
//!
@@ -278,8 +278,10 @@
cfg_rt! {
pub use crate::runtime::task::{JoinError, JoinHandle};
- mod blocking;
- pub use blocking::spawn_blocking;
+ cfg_not_wasi! {
+ mod blocking;
+ pub use blocking::spawn_blocking;
+ }
mod spawn;
pub use spawn::spawn;
@@ -291,8 +293,13 @@
mod yield_now;
pub use yield_now::yield_now;
+ cfg_unstable! {
+ mod consume_budget;
+ pub use consume_budget::consume_budget;
+ }
+
mod local;
- pub use local::{spawn_local, LocalSet};
+ pub use local::{spawn_local, LocalSet, LocalEnterGuard};
mod task_local;
pub use task_local::LocalKey;
@@ -300,6 +307,20 @@
mod unconstrained;
pub use unconstrained::{unconstrained, Unconstrained};
+ #[doc(inline)]
+ pub use join_set::JoinSet;
+ pub use crate::runtime::task::AbortHandle;
+
+ // Uses #[cfg(...)] instead of macro since the macro adds docsrs annotations.
+ #[cfg(not(tokio_unstable))]
+ mod join_set;
+ #[cfg(tokio_unstable)]
+ pub mod join_set;
+
+ cfg_unstable! {
+ pub use crate::runtime::task::{Id, id, try_id};
+ }
+
cfg_trace! {
mod builder;
pub use builder::Builder;
diff --git a/src/task/spawn.rs b/src/task/spawn.rs
index 065d38f..66b0d67 100644
--- a/src/task/spawn.rs
+++ b/src/task/spawn.rs
@@ -1,4 +1,5 @@
-use crate::{task::JoinHandle, util::error::CONTEXT_MISSING_ERROR};
+use crate::runtime::Handle;
+use crate::task::JoinHandle;
use std::future::Future;
@@ -6,6 +7,10 @@
/// Spawns a new asynchronous task, returning a
/// [`JoinHandle`](super::JoinHandle) for it.
///
+ /// The provided future will start running in the background immediately
+ /// when `spawn` is called, even if you don't await the returned
+ /// `JoinHandle`.
+ ///
/// Spawning a task enables the task to execute concurrently to other tasks. The
/// spawned task may execute on the current thread, or it may be sent to a
/// different thread to be executed. The specifics depend on the current
@@ -49,6 +54,37 @@
/// }
/// ```
///
+ /// To run multiple tasks in parallel and receive their results, join
+ /// handles can be stored in a vector.
+ /// ```
+ /// # #[tokio::main(flavor = "current_thread")] async fn main() {
+ /// async fn my_background_op(id: i32) -> String {
+ /// let s = format!("Starting background task {}.", id);
+ /// println!("{}", s);
+ /// s
+ /// }
+ ///
+ /// let ops = vec![1, 2, 3];
+ /// let mut tasks = Vec::with_capacity(ops.len());
+ /// for op in ops {
+ /// // This call will make them start running in the background
+ /// // immediately.
+ /// tasks.push(tokio::spawn(my_background_op(op)));
+ /// }
+ ///
+ /// let mut outputs = Vec::with_capacity(tasks.len());
+ /// for task in tasks {
+ /// outputs.push(task.await.unwrap());
+ /// }
+ /// println!("{:?}", outputs);
+ /// # }
+ /// ```
+ /// This example pushes the tasks to `outputs` in the order they were
+ /// started in. If you do not care about the ordering of the outputs, then
+ /// you can also use a [`JoinSet`].
+ ///
+ /// [`JoinSet`]: struct@crate::task::JoinSet
+ ///
/// # Panics
///
/// Panics if called from **outside** of the Tokio runtime.
@@ -121,7 +157,7 @@
/// ```text
/// error[E0391]: cycle detected when processing `main`
/// ```
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub fn spawn<T>(future: T) -> JoinHandle<T::Output>
where
T: Future + Send + 'static,
@@ -136,14 +172,16 @@
}
}
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub(super) fn spawn_inner<T>(future: T, name: Option<&str>) -> JoinHandle<T::Output>
where
T: Future + Send + 'static,
T::Output: Send + 'static,
{
- let spawn_handle = crate::runtime::context::spawn_handle().expect(CONTEXT_MISSING_ERROR);
- let task = crate::util::trace::task(future, "task", name);
- spawn_handle.spawn(task)
+ use crate::runtime::task;
+ let id = task::Id::next();
+ let task = crate::util::trace::task(future, "task", name, id.as_u64());
+ let handle = Handle::current();
+ handle.inner.spawn(task, id)
}
}
diff --git a/src/task/task_local.rs b/src/task/task_local.rs
index b6e7df4..d3b108f 100644
--- a/src/task/task_local.rs
+++ b/src/task/task_local.rs
@@ -1,11 +1,10 @@
-use pin_project_lite::pin_project;
use std::cell::RefCell;
use std::error::Error;
use std::future::Future;
use std::marker::PhantomPinned;
use std::pin::Pin;
use std::task::{Context, Poll};
-use std::{fmt, thread};
+use std::{fmt, mem, thread};
/// Declares a new task-local key of type [`tokio::task::LocalKey`].
///
@@ -48,9 +47,27 @@
}
#[doc(hidden)]
+#[cfg(not(tokio_no_const_thread_local))]
#[macro_export]
macro_rules! __task_local_inner {
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty) => {
+ $(#[$attr])*
+ $vis static $name: $crate::task::LocalKey<$t> = {
+ std::thread_local! {
+ static __KEY: std::cell::RefCell<Option<$t>> = const { std::cell::RefCell::new(None) };
+ }
+
+ $crate::task::LocalKey { inner: __KEY }
+ };
+ };
+}
+
+#[doc(hidden)]
+#[cfg(tokio_no_const_thread_local)]
+#[macro_export]
+macro_rules! __task_local_inner {
+ ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty) => {
+ $(#[$attr])*
$vis static $name: $crate::task::LocalKey<$t> = {
std::thread_local! {
static __KEY: std::cell::RefCell<Option<$t>> = std::cell::RefCell::new(None);
@@ -63,7 +80,7 @@
/// A key for task-local data.
///
-/// This type is generated by the `task_local!` macro.
+/// This type is generated by the [`task_local!`] macro.
///
/// Unlike [`std::thread::LocalKey`], `tokio::task::LocalKey` will
/// _not_ lazily initialize the value on first access. Instead, the
@@ -91,7 +108,9 @@
/// }).await;
/// # }
/// ```
+///
/// [`std::thread::LocalKey`]: struct@std::thread::LocalKey
+/// [`task_local!`]: ../macro.task_local.html
#[cfg_attr(docsrs, doc(cfg(feature = "rt")))]
pub struct LocalKey<T: 'static> {
#[doc(hidden)]
@@ -103,6 +122,11 @@
///
/// On completion of `scope`, the task-local will be dropped.
///
+ /// ### Panics
+ ///
+ /// If you poll the returned future inside a call to [`with`] or
+ /// [`try_with`] on the same `LocalKey`, then the call to `poll` will panic.
+ ///
/// ### Examples
///
/// ```
@@ -116,6 +140,9 @@
/// }).await;
/// # }
/// ```
+ ///
+ /// [`with`]: fn@Self::with
+ /// [`try_with`]: fn@Self::try_with
pub fn scope<F>(&'static self, value: T, f: F) -> TaskLocalFuture<T, F>
where
F: Future,
@@ -123,14 +150,19 @@
TaskLocalFuture {
local: self,
slot: Some(value),
- future: f,
+ future: Some(f),
_pinned: PhantomPinned,
}
}
/// Sets a value `T` as the task-local value for the closure `F`.
///
- /// On completion of `scope`, the task-local will be dropped.
+ /// On completion of `sync_scope`, the task-local will be dropped.
+ ///
+ /// ### Panics
+ ///
+ /// This method panics if called inside a call to [`with`] or [`try_with`]
+ /// on the same `LocalKey`.
///
/// ### Examples
///
@@ -145,34 +177,80 @@
/// });
/// # }
/// ```
+ ///
+ /// [`with`]: fn@Self::with
+ /// [`try_with`]: fn@Self::try_with
+ #[track_caller]
pub fn sync_scope<F, R>(&'static self, value: T, f: F) -> R
where
F: FnOnce() -> R,
{
- let scope = TaskLocalFuture {
- local: self,
- slot: Some(value),
- future: (),
- _pinned: PhantomPinned,
- };
- crate::pin!(scope);
- scope.with_task(|_| f())
+ let mut value = Some(value);
+ match self.scope_inner(&mut value, f) {
+ Ok(res) => res,
+ Err(err) => err.panic(),
+ }
+ }
+
+ fn scope_inner<F, R>(&'static self, slot: &mut Option<T>, f: F) -> Result<R, ScopeInnerErr>
+ where
+ F: FnOnce() -> R,
+ {
+ struct Guard<'a, T: 'static> {
+ local: &'static LocalKey<T>,
+ slot: &'a mut Option<T>,
+ }
+
+ impl<'a, T: 'static> Drop for Guard<'a, T> {
+ fn drop(&mut self) {
+ // This should not panic.
+ //
+ // We know that the RefCell was not borrowed before the call to
+ // `scope_inner`, so the only way for this to panic is if the
+ // closure has created but not destroyed a RefCell guard.
+ // However, we never give user-code access to the guards, so
+ // there's no way for user-code to forget to destroy a guard.
+ //
+ // The call to `with` also should not panic, since the
+ // thread-local wasn't destroyed when we first called
+ // `scope_inner`, and it shouldn't have gotten destroyed since
+ // then.
+ self.local.inner.with(|inner| {
+ let mut ref_mut = inner.borrow_mut();
+ mem::swap(self.slot, &mut *ref_mut);
+ });
+ }
+ }
+
+ self.inner.try_with(|inner| {
+ inner
+ .try_borrow_mut()
+ .map(|mut ref_mut| mem::swap(slot, &mut *ref_mut))
+ })??;
+
+ let guard = Guard { local: self, slot };
+
+ let res = f();
+
+ drop(guard);
+
+ Ok(res)
}
/// Accesses the current task-local and runs the provided closure.
///
/// # Panics
///
- /// This function will panic if not called within the context
- /// of a future containing a task-local with the corresponding key.
+ /// This function will panic if the task local doesn't have a value set.
+ #[track_caller]
pub fn with<F, R>(&'static self, f: F) -> R
where
F: FnOnce(&T) -> R,
{
- self.try_with(f).expect(
- "cannot access a Task Local Storage value \
- without setting it via `LocalKey::set`",
- )
+ match self.try_with(f) {
+ Ok(res) => res,
+ Err(_) => panic!("cannot access a task-local storage value without setting it first"),
+ }
}
/// Accesses the current task-local and runs the provided closure.
@@ -184,19 +262,32 @@
where
F: FnOnce(&T) -> R,
{
- self.inner.with(|v| {
- if let Some(val) = v.borrow().as_ref() {
- Ok(f(val))
- } else {
- Err(AccessError { _private: () })
- }
- })
+ // If called after the thread-local storing the task-local is destroyed,
+ // then we are outside of a closure where the task-local is set.
+ //
+ // Therefore, it is correct to return an AccessError if `try_with`
+ // returns an error.
+ let try_with_res = self.inner.try_with(|v| {
+ // This call to `borrow` cannot panic because no user-defined code
+ // runs while a `borrow_mut` call is active.
+ v.borrow().as_ref().map(f)
+ });
+
+ match try_with_res {
+ Ok(Some(res)) => Ok(res),
+ Ok(None) | Err(_) => Err(AccessError { _private: () }),
+ }
}
}
impl<T: Copy + 'static> LocalKey<T> {
/// Returns a copy of the task-local value
/// if the task-local value implements `Copy`.
+ ///
+ /// # Panics
+ ///
+ /// This function will panic if the task local doesn't have a value set.
+ #[track_caller]
pub fn get(&'static self) -> T {
self.with(|v| *v)
}
@@ -208,76 +299,104 @@
}
}
-pin_project! {
- /// A future that sets a value `T` of a task local for the future `F` during
- /// its execution.
- ///
- /// The value of the task-local must be `'static` and will be dropped on the
- /// completion of the future.
- ///
- /// Created by the function [`LocalKey::scope`](self::LocalKey::scope).
- ///
- /// ### Examples
- ///
- /// ```
- /// # async fn dox() {
- /// tokio::task_local! {
- /// static NUMBER: u32;
- /// }
- ///
- /// NUMBER.scope(1, async move {
- /// println!("task local value: {}", NUMBER.get());
- /// }).await;
- /// # }
- /// ```
- pub struct TaskLocalFuture<T, F>
- where
- T: 'static
- {
- local: &'static LocalKey<T>,
- slot: Option<T>,
- #[pin]
- future: F,
- #[pin]
- _pinned: PhantomPinned,
- }
-}
-
-impl<T: 'static, F> TaskLocalFuture<T, F> {
- fn with_task<F2: FnOnce(Pin<&mut F>) -> R, R>(self: Pin<&mut Self>, f: F2) -> R {
- struct Guard<'a, T: 'static> {
- local: &'static LocalKey<T>,
- slot: &'a mut Option<T>,
- prev: Option<T>,
- }
-
- impl<T> Drop for Guard<'_, T> {
- fn drop(&mut self) {
- let value = self.local.inner.with(|c| c.replace(self.prev.take()));
- *self.slot = value;
- }
- }
-
- let mut project = self.project();
- let val = project.slot.take();
-
- let prev = project.local.inner.with(|c| c.replace(val));
-
- let _guard = Guard {
- prev,
- slot: &mut project.slot,
- local: *project.local,
- };
-
- f(project.future)
- }
+/// A future that sets a value `T` of a task local for the future `F` during
+/// its execution.
+///
+/// The value of the task-local must be `'static` and will be dropped on the
+/// completion of the future.
+///
+/// Created by the function [`LocalKey::scope`](self::LocalKey::scope).
+///
+/// ### Examples
+///
+/// ```
+/// # async fn dox() {
+/// tokio::task_local! {
+/// static NUMBER: u32;
+/// }
+///
+/// NUMBER.scope(1, async move {
+/// println!("task local value: {}", NUMBER.get());
+/// }).await;
+/// # }
+/// ```
+// Doesn't use pin_project due to custom Drop.
+pub struct TaskLocalFuture<T, F>
+where
+ T: 'static,
+{
+ local: &'static LocalKey<T>,
+ slot: Option<T>,
+ future: Option<F>,
+ _pinned: PhantomPinned,
}
impl<T: 'static, F: Future> Future for TaskLocalFuture<T, F> {
type Output = F::Output;
+ #[track_caller]
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
- self.with_task(|f| f.poll(cx))
+ // safety: The TaskLocalFuture struct is `!Unpin` so there is no way to
+ // move `self.future` from now on.
+ let this = unsafe { Pin::into_inner_unchecked(self) };
+ let mut future_opt = unsafe { Pin::new_unchecked(&mut this.future) };
+
+ let res =
+ this.local
+ .scope_inner(&mut this.slot, || match future_opt.as_mut().as_pin_mut() {
+ Some(fut) => {
+ let res = fut.poll(cx);
+ if res.is_ready() {
+ future_opt.set(None);
+ }
+ Some(res)
+ }
+ None => None,
+ });
+
+ match res {
+ Ok(Some(res)) => res,
+ Ok(None) => panic!("`TaskLocalFuture` polled after completion"),
+ Err(err) => err.panic(),
+ }
+ }
+}
+
+impl<T: 'static, F> Drop for TaskLocalFuture<T, F> {
+ fn drop(&mut self) {
+ if mem::needs_drop::<F>() && self.future.is_some() {
+ // Drop the future while the task-local is set, if possible. Otherwise
+ // the future is dropped normally when the `Option<F>` field drops.
+ let future = &mut self.future;
+ let _ = self.local.scope_inner(&mut self.slot, || {
+ *future = None;
+ });
+ }
+ }
+}
+
+impl<T: 'static, F> fmt::Debug for TaskLocalFuture<T, F>
+where
+ T: fmt::Debug,
+{
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ /// Format the Option without Some.
+ struct TransparentOption<'a, T> {
+ value: &'a Option<T>,
+ }
+ impl<'a, T: fmt::Debug> fmt::Debug for TransparentOption<'a, T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self.value.as_ref() {
+ Some(value) => value.fmt(f),
+ // Hitting the None branch should not be possible.
+ None => f.pad("<missing>"),
+ }
+ }
+ }
+
+ f.debug_struct("TaskLocalFuture")
+ .field("value", &TransparentOption { value: &self.slot })
+ .finish()
}
}
@@ -300,3 +419,30 @@
}
impl Error for AccessError {}
+
+enum ScopeInnerErr {
+ BorrowError,
+ AccessError,
+}
+
+impl ScopeInnerErr {
+ #[track_caller]
+ fn panic(&self) -> ! {
+ match self {
+ Self::BorrowError => panic!("cannot enter a task-local scope while the task-local storage is borrowed"),
+ Self::AccessError => panic!("cannot enter a task-local scope during or after destruction of the underlying thread-local"),
+ }
+ }
+}
+
+impl From<std::cell::BorrowMutError> for ScopeInnerErr {
+ fn from(_: std::cell::BorrowMutError) -> Self {
+ Self::BorrowError
+ }
+}
+
+impl From<std::thread::AccessError> for ScopeInnerErr {
+ fn from(_: std::thread::AccessError) -> Self {
+ Self::AccessError
+ }
+}
diff --git a/src/task/unconstrained.rs b/src/task/unconstrained.rs
index 31c732b..40384c8 100644
--- a/src/task/unconstrained.rs
+++ b/src/task/unconstrained.rs
@@ -22,7 +22,7 @@
cfg_coop! {
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let inner = self.project().inner;
- crate::coop::with_unconstrained(|| inner.poll(cx))
+ crate::runtime::coop::with_unconstrained(|| inner.poll(cx))
}
}
diff --git a/src/task/yield_now.rs b/src/task/yield_now.rs
index 5eeb46a..7b61dd8 100644
--- a/src/task/yield_now.rs
+++ b/src/task/yield_now.rs
@@ -1,3 +1,5 @@
+use crate::runtime::context;
+
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
@@ -33,7 +35,6 @@
/// which order the runtime polls your tasks in.
///
/// [`tokio::select!`]: macro@crate::select
-#[must_use = "yield_now does nothing unless polled/`await`-ed"]
#[cfg_attr(docsrs, doc(cfg(feature = "rt")))]
pub async fn yield_now() {
/// Yield implementation
@@ -50,7 +51,17 @@
}
self.yielded = true;
- cx.waker().wake_by_ref();
+
+ let defer = context::with_defer(|rt| {
+ rt.defer(cx.waker().clone());
+ });
+
+ if defer.is_none() {
+ // Not currently in a runtime, just notify ourselves
+ // immediately.
+ cx.waker().wake_by_ref();
+ }
+
Poll::Pending
}
}
diff --git a/src/time/clock.rs b/src/time/clock.rs
index 41be9ba..cd11a67 100644
--- a/src/time/clock.rs
+++ b/src/time/clock.rs
@@ -33,7 +33,13 @@
cfg_rt! {
fn clock() -> Option<Clock> {
- crate::runtime::context::clock()
+ use crate::runtime::Handle;
+
+ match Handle::try_current() {
+ Ok(handle) => Some(handle.inner.driver().clock().clone()),
+ Err(ref e) if e.is_missing_context() => None,
+ Err(_) => panic!("{}", crate::util::error::THREAD_LOCAL_DESTROYED_ERROR),
+ }
}
}
@@ -59,6 +65,9 @@
/// Instant at which the clock was last unfrozen.
unfrozen: Option<std::time::Instant>,
+
+ /// Number of `inhibit_auto_advance` calls still in effect.
+ auto_advance_inhibit_count: usize,
}
/// Pauses time.
@@ -96,6 +105,7 @@
///
/// [`Sleep`]: crate::time::Sleep
/// [`advance`]: crate::time::advance
+ #[track_caller]
pub fn pause() {
let clock = clock().expect("time cannot be frozen from outside the Tokio runtime");
clock.pause();
@@ -110,6 +120,7 @@
///
/// Panics if time is not frozen or if called from outside of the Tokio
/// runtime.
+ #[track_caller]
pub fn resume() {
let clock = clock().expect("time cannot be frozen from outside the Tokio runtime");
let mut inner = clock.inner.lock();
@@ -179,6 +190,7 @@
enable_pausing,
base: now,
unfrozen: Some(now),
+ auto_advance_inhibit_count: 0,
})),
};
@@ -189,6 +201,7 @@
clock
}
+ #[track_caller]
pub(crate) fn pause(&self) {
let mut inner = self.inner.lock();
@@ -203,11 +216,23 @@
inner.unfrozen = None;
}
- pub(crate) fn is_paused(&self) -> bool {
- let inner = self.inner.lock();
- inner.unfrozen.is_none()
+ /// Temporarily stop auto-advancing the clock (see `tokio::time::pause`).
+ pub(crate) fn inhibit_auto_advance(&self) {
+ let mut inner = self.inner.lock();
+ inner.auto_advance_inhibit_count += 1;
}
+ pub(crate) fn allow_auto_advance(&self) {
+ let mut inner = self.inner.lock();
+ inner.auto_advance_inhibit_count -= 1;
+ }
+
+ pub(crate) fn can_auto_advance(&self) -> bool {
+ let inner = self.inner.lock();
+ inner.unfrozen.is_none() && inner.auto_advance_inhibit_count == 0
+ }
+
+ #[track_caller]
pub(crate) fn advance(&self, duration: Duration) {
let mut inner = self.inner.lock();
diff --git a/src/time/driver/handle.rs b/src/time/driver/handle.rs
deleted file mode 100644
index 7aaf65a..0000000
--- a/src/time/driver/handle.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-use crate::loom::sync::Arc;
-use crate::time::driver::ClockTime;
-use std::fmt;
-
-/// Handle to time driver instance.
-#[derive(Clone)]
-pub(crate) struct Handle {
- time_source: ClockTime,
- inner: Arc<super::Inner>,
-}
-
-impl Handle {
- /// Creates a new timer `Handle` from a shared `Inner` timer state.
- pub(super) fn new(inner: Arc<super::Inner>) -> Self {
- let time_source = inner.state.lock().time_source.clone();
- Handle { time_source, inner }
- }
-
- /// Returns the time source associated with this handle.
- pub(super) fn time_source(&self) -> &ClockTime {
- &self.time_source
- }
-
- /// Access the driver's inner structure.
- pub(super) fn get(&self) -> &super::Inner {
- &*self.inner
- }
-
- /// Checks whether the driver has been shutdown.
- pub(super) fn is_shutdown(&self) -> bool {
- self.inner.is_shutdown()
- }
-}
-
-cfg_rt! {
- impl Handle {
- /// Tries to get a handle to the current timer.
- ///
- /// # Panics
- ///
- /// This function panics if there is no current timer set.
- ///
- /// It can be triggered when `Builder::enable_time()` or
- /// `Builder::enable_all()` are not included in the builder.
- ///
- /// It can also panic whenever a timer is created outside of a
- /// Tokio runtime. That is why `rt.block_on(delay_for(...))` will panic,
- /// since the function is executed outside of the runtime.
- /// Whereas `rt.block_on(async {delay_for(...).await})` doesn't panic.
- /// And this is because wrapping the function on an async makes it lazy,
- /// and so gets executed inside the runtime successfully without
- /// panicking.
- pub(crate) fn current() -> Self {
- crate::runtime::context::time_handle()
- .expect("A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers.")
- }
- }
-}
-
-cfg_not_rt! {
- impl Handle {
- /// Tries to get a handle to the current timer.
- ///
- /// # Panics
- ///
- /// This function panics if there is no current timer set.
- ///
- /// It can be triggered when `Builder::enable_time()` or
- /// `Builder::enable_all()` are not included in the builder.
- ///
- /// It can also panic whenever a timer is created outside of a Tokio
- /// runtime. That is why `rt.block_on(delay_for(...))` will panic,
- /// since the function is executed outside of the runtime.
- /// Whereas `rt.block_on(async {delay_for(...).await})` doesn't
- /// panic. And this is because wrapping the function on an async makes it
- /// lazy, and so outside executed inside the runtime successfully without
- /// panicking.
- pub(crate) fn current() -> Self {
- panic!("{}", crate::util::error::CONTEXT_MISSING_ERROR)
- }
- }
-}
-
-impl fmt::Debug for Handle {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- write!(f, "Handle")
- }
-}
diff --git a/src/time/driver/tests/mod.rs b/src/time/driver/tests/mod.rs
deleted file mode 100644
index 7c5cf1f..0000000
--- a/src/time/driver/tests/mod.rs
+++ /dev/null
@@ -1,287 +0,0 @@
-use std::{task::Context, time::Duration};
-
-#[cfg(not(loom))]
-use futures::task::noop_waker_ref;
-
-use crate::loom::sync::Arc;
-use crate::loom::thread;
-use crate::{
- loom::sync::atomic::{AtomicBool, Ordering},
- park::Unpark,
-};
-
-use super::{Handle, TimerEntry};
-
-struct MockUnpark {}
-impl Unpark for MockUnpark {
- fn unpark(&self) {}
-}
-impl MockUnpark {
- fn mock() -> Box<dyn Unpark> {
- Box::new(Self {})
- }
-}
-
-fn block_on<T>(f: impl std::future::Future<Output = T>) -> T {
- #[cfg(loom)]
- return loom::future::block_on(f);
-
- #[cfg(not(loom))]
- return futures::executor::block_on(f);
-}
-
-fn model(f: impl Fn() + Send + Sync + 'static) {
- #[cfg(loom)]
- loom::model(f);
-
- #[cfg(not(loom))]
- f();
-}
-
-#[test]
-fn single_timer() {
- model(|| {
- let clock = crate::time::clock::Clock::new(true, false);
- let time_source = super::ClockTime::new(clock.clone());
-
- let inner = super::Inner::new(time_source.clone(), MockUnpark::mock());
- let handle = Handle::new(Arc::new(inner));
-
- let handle_ = handle.clone();
- let jh = thread::spawn(move || {
- let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1));
- pin!(entry);
-
- block_on(futures::future::poll_fn(|cx| {
- entry.as_mut().poll_elapsed(cx)
- }))
- .unwrap();
- });
-
- thread::yield_now();
-
- // This may or may not return Some (depending on how it races with the
- // thread). If it does return None, however, the timer should complete
- // synchronously.
- handle.process_at_time(time_source.now() + 2_000_000_000);
-
- jh.join().unwrap();
- })
-}
-
-#[test]
-fn drop_timer() {
- model(|| {
- let clock = crate::time::clock::Clock::new(true, false);
- let time_source = super::ClockTime::new(clock.clone());
-
- let inner = super::Inner::new(time_source.clone(), MockUnpark::mock());
- let handle = Handle::new(Arc::new(inner));
-
- let handle_ = handle.clone();
- let jh = thread::spawn(move || {
- let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1));
- pin!(entry);
-
- let _ = entry
- .as_mut()
- .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
- let _ = entry
- .as_mut()
- .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
- });
-
- thread::yield_now();
-
- // advance 2s in the future.
- handle.process_at_time(time_source.now() + 2_000_000_000);
-
- jh.join().unwrap();
- })
-}
-
-#[test]
-fn change_waker() {
- model(|| {
- let clock = crate::time::clock::Clock::new(true, false);
- let time_source = super::ClockTime::new(clock.clone());
-
- let inner = super::Inner::new(time_source.clone(), MockUnpark::mock());
- let handle = Handle::new(Arc::new(inner));
-
- let handle_ = handle.clone();
- let jh = thread::spawn(move || {
- let entry = TimerEntry::new(&handle_, clock.now() + Duration::from_secs(1));
- pin!(entry);
-
- let _ = entry
- .as_mut()
- .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
-
- block_on(futures::future::poll_fn(|cx| {
- entry.as_mut().poll_elapsed(cx)
- }))
- .unwrap();
- });
-
- thread::yield_now();
-
- // advance 2s
- handle.process_at_time(time_source.now() + 2_000_000_000);
-
- jh.join().unwrap();
- })
-}
-
-#[test]
-fn reset_future() {
- model(|| {
- let finished_early = Arc::new(AtomicBool::new(false));
-
- let clock = crate::time::clock::Clock::new(true, false);
- let time_source = super::ClockTime::new(clock.clone());
-
- let inner = super::Inner::new(time_source.clone(), MockUnpark::mock());
- let handle = Handle::new(Arc::new(inner));
-
- let handle_ = handle.clone();
- let finished_early_ = finished_early.clone();
- let start = clock.now();
-
- let jh = thread::spawn(move || {
- let entry = TimerEntry::new(&handle_, start + Duration::from_secs(1));
- pin!(entry);
-
- let _ = entry
- .as_mut()
- .poll_elapsed(&mut Context::from_waker(futures::task::noop_waker_ref()));
-
- entry.as_mut().reset(start + Duration::from_secs(2));
-
- // shouldn't complete before 2s
- block_on(futures::future::poll_fn(|cx| {
- entry.as_mut().poll_elapsed(cx)
- }))
- .unwrap();
-
- finished_early_.store(true, Ordering::Relaxed);
- });
-
- thread::yield_now();
-
- // This may or may not return a wakeup time.
- handle.process_at_time(time_source.instant_to_tick(start + Duration::from_millis(1500)));
-
- assert!(!finished_early.load(Ordering::Relaxed));
-
- handle.process_at_time(time_source.instant_to_tick(start + Duration::from_millis(2500)));
-
- jh.join().unwrap();
-
- assert!(finished_early.load(Ordering::Relaxed));
- })
-}
-
-#[test]
-#[cfg(not(loom))]
-fn poll_process_levels() {
- let clock = crate::time::clock::Clock::new(true, false);
- clock.pause();
-
- let time_source = super::ClockTime::new(clock.clone());
-
- let inner = super::Inner::new(time_source, MockUnpark::mock());
- let handle = Handle::new(Arc::new(inner));
-
- let mut entries = vec![];
-
- for i in 0..1024 {
- let mut entry = Box::pin(TimerEntry::new(
- &handle,
- clock.now() + Duration::from_millis(i),
- ));
-
- let _ = entry
- .as_mut()
- .poll_elapsed(&mut Context::from_waker(noop_waker_ref()));
-
- entries.push(entry);
- }
-
- for t in 1..1024 {
- handle.process_at_time(t as u64);
- for (deadline, future) in entries.iter_mut().enumerate() {
- let mut context = Context::from_waker(noop_waker_ref());
- if deadline <= t {
- assert!(future.as_mut().poll_elapsed(&mut context).is_ready());
- } else {
- assert!(future.as_mut().poll_elapsed(&mut context).is_pending());
- }
- }
- }
-}
-
-#[test]
-#[cfg(not(loom))]
-fn poll_process_levels_targeted() {
- let mut context = Context::from_waker(noop_waker_ref());
-
- let clock = crate::time::clock::Clock::new(true, false);
- clock.pause();
-
- let time_source = super::ClockTime::new(clock.clone());
-
- let inner = super::Inner::new(time_source, MockUnpark::mock());
- let handle = Handle::new(Arc::new(inner));
-
- let e1 = TimerEntry::new(&handle, clock.now() + Duration::from_millis(193));
- pin!(e1);
-
- handle.process_at_time(62);
- assert!(e1.as_mut().poll_elapsed(&mut context).is_pending());
- handle.process_at_time(192);
- handle.process_at_time(192);
-}
-
-/*
-#[test]
-fn balanced_incr_and_decr() {
- const OPS: usize = 5;
-
- fn incr(inner: Arc<Inner>) {
- for _ in 0..OPS {
- inner.increment().expect("increment should not have failed");
- thread::yield_now();
- }
- }
-
- fn decr(inner: Arc<Inner>) {
- let mut ops_performed = 0;
- while ops_performed < OPS {
- if inner.num(Ordering::Relaxed) > 0 {
- ops_performed += 1;
- inner.decrement();
- }
- thread::yield_now();
- }
- }
-
- loom::model(|| {
- let unpark = Box::new(MockUnpark);
- let instant = Instant::now();
-
- let inner = Arc::new(Inner::new(instant, unpark));
-
- let incr_inner = inner.clone();
- let decr_inner = inner.clone();
-
- let incr_hndle = thread::spawn(move || incr(incr_inner));
- let decr_hndle = thread::spawn(move || decr(decr_inner));
-
- incr_hndle.join().expect("should never fail");
- decr_hndle.join().expect("should never fail");
-
- assert_eq!(inner.num(Ordering::SeqCst), 0);
- })
-}
-*/
diff --git a/src/time/driver/wheel/stack.rs b/src/time/driver/wheel/stack.rs
deleted file mode 100644
index 80651c3..0000000
--- a/src/time/driver/wheel/stack.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-use super::{Item, OwnedItem};
-use crate::time::driver::Entry;
-
-use std::ptr;
-
-/// A doubly linked stack.
-#[derive(Debug)]
-pub(crate) struct Stack {
- head: Option<OwnedItem>,
-}
-
-impl Default for Stack {
- fn default() -> Stack {
- Stack { head: None }
- }
-}
-
-impl Stack {
- pub(crate) fn is_empty(&self) -> bool {
- self.head.is_none()
- }
-
- pub(crate) fn push(&mut self, entry: OwnedItem) {
- // Get a pointer to the entry to for the prev link
- let ptr: *const Entry = &*entry as *const _;
-
- // Remove the old head entry
- let old = self.head.take();
-
- unsafe {
- // Ensure the entry is not already in a stack.
- debug_assert!((*entry.next_stack.get()).is_none());
- debug_assert!((*entry.prev_stack.get()).is_null());
-
- if let Some(ref entry) = old.as_ref() {
- debug_assert!({
- // The head is not already set to the entry
- ptr != &***entry as *const _
- });
-
- // Set the previous link on the old head
- *entry.prev_stack.get() = ptr;
- }
-
- // Set this entry's next pointer
- *entry.next_stack.get() = old;
- }
-
- // Update the head pointer
- self.head = Some(entry);
- }
-
- /// Pops an item from the stack.
- pub(crate) fn pop(&mut self) -> Option<OwnedItem> {
- let entry = self.head.take();
-
- unsafe {
- if let Some(entry) = entry.as_ref() {
- self.head = (*entry.next_stack.get()).take();
-
- if let Some(entry) = self.head.as_ref() {
- *entry.prev_stack.get() = ptr::null();
- }
-
- *entry.prev_stack.get() = ptr::null();
- }
- }
-
- entry
- }
-
- pub(crate) fn remove(&mut self, entry: &Item) {
- unsafe {
- // Ensure that the entry is in fact contained by the stack
- debug_assert!({
- // This walks the full linked list even if an entry is found.
- let mut next = self.head.as_ref();
- let mut contains = false;
-
- while let Some(n) = next {
- if entry as *const _ == &**n as *const _ {
- debug_assert!(!contains);
- contains = true;
- }
-
- next = (*n.next_stack.get()).as_ref();
- }
-
- contains
- });
-
- // Unlink `entry` from the next node
- let next = (*entry.next_stack.get()).take();
-
- if let Some(next) = next.as_ref() {
- (*next.prev_stack.get()) = *entry.prev_stack.get();
- }
-
- // Unlink `entry` from the prev node
-
- if let Some(prev) = (*entry.prev_stack.get()).as_ref() {
- *prev.next_stack.get() = next;
- } else {
- // It is the head
- self.head = next;
- }
-
- // Unset the prev pointer
- *entry.prev_stack.get() = ptr::null();
- }
- }
-}
diff --git a/src/time/error.rs b/src/time/error.rs
index 63f0a3b..71344d4 100644
--- a/src/time/error.rs
+++ b/src/time/error.rs
@@ -41,7 +41,10 @@
}
/// Errors returned by `Timeout`.
-#[derive(Debug, PartialEq)]
+///
+/// This error is returned when a timeout expires before the function was able
+/// to finish.
+#[derive(Debug, PartialEq, Eq)]
pub struct Elapsed(());
#[derive(Debug)]
diff --git a/src/time/instant.rs b/src/time/instant.rs
index f7cf12d..f184929 100644
--- a/src/time/instant.rs
+++ b/src/time/instant.rs
@@ -67,13 +67,10 @@
self.std
}
- /// Returns the amount of time elapsed from another instant to this one.
- ///
- /// # Panics
- ///
- /// This function will panic if `earlier` is later than `self`.
+ /// Returns the amount of time elapsed from another instant to this one, or
+ /// zero duration if that instant is later than this one.
pub fn duration_since(&self, earlier: Instant) -> Duration {
- self.std.duration_since(earlier.std)
+ self.std.saturating_duration_since(earlier.std)
}
/// Returns the amount of time elapsed from another instant to this one, or
@@ -118,13 +115,8 @@
self.std.saturating_duration_since(earlier.std)
}
- /// Returns the amount of time elapsed since this instant was created.
- ///
- /// # Panics
- ///
- /// This function may panic if the current time is earlier than this
- /// instant, which is something that can happen if an `Instant` is
- /// produced synthetically.
+ /// Returns the amount of time elapsed since this instant was created,
+ /// or zero duration if that this instant is in the future.
///
/// # Examples
///
@@ -140,7 +132,7 @@
/// }
/// ```
pub fn elapsed(&self) -> Duration {
- Instant::now() - *self
+ Instant::now().saturating_duration_since(*self)
}
/// Returns `Some(t)` where `t` is the time `self + duration` if `t` can be
@@ -188,7 +180,7 @@
type Output = Duration;
fn sub(self, rhs: Instant) -> Duration {
- self.std - rhs.std
+ self.std.saturating_duration_since(rhs.std)
}
}
diff --git a/src/time/interval.rs b/src/time/interval.rs
index 7e07e51..ea8b393 100644
--- a/src/time/interval.rs
+++ b/src/time/interval.rs
@@ -1,6 +1,8 @@
use crate::future::poll_fn;
use crate::time::{sleep_until, Duration, Instant, Sleep};
+use crate::util::trace;
+use std::panic::Location;
use std::pin::Pin;
use std::task::{Context, Poll};
use std::{convert::TryInto, future::Future};
@@ -68,10 +70,10 @@
///
/// [`sleep`]: crate::time::sleep()
/// [`.tick().await`]: Interval::tick
+#[track_caller]
pub fn interval(period: Duration) -> Interval {
assert!(period > Duration::new(0, 0), "`period` must be non-zero.");
-
- interval_at(Instant::now(), period)
+ internal_interval_at(Instant::now(), period, trace::caller_location())
}
/// Creates new [`Interval`] that yields with interval of `period` with the
@@ -103,13 +105,44 @@
/// // approximately 70ms have elapsed.
/// }
/// ```
+#[track_caller]
pub fn interval_at(start: Instant, period: Duration) -> Interval {
assert!(period > Duration::new(0, 0), "`period` must be non-zero.");
+ internal_interval_at(start, period, trace::caller_location())
+}
+
+#[cfg_attr(not(all(tokio_unstable, feature = "tracing")), allow(unused_variables))]
+fn internal_interval_at(
+ start: Instant,
+ period: Duration,
+ location: Option<&'static Location<'static>>,
+) -> Interval {
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = {
+ let location = location.expect("should have location if tracing");
+
+ tracing::trace_span!(
+ "runtime.resource",
+ concrete_type = "Interval",
+ kind = "timer",
+ loc.file = location.file(),
+ loc.line = location.line(),
+ loc.col = location.column(),
+ )
+ };
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let delay = resource_span.in_scope(|| Box::pin(sleep_until(start)));
+
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let delay = Box::pin(sleep_until(start));
Interval {
- delay: Box::pin(sleep_until(start)),
+ delay,
period,
missed_tick_behavior: Default::default(),
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span,
}
}
@@ -124,7 +157,7 @@
///
/// #[tokio::main]
/// async fn main() {
-/// // ticks every 2 seconds
+/// // ticks every 2 milliseconds
/// let mut interval = time::interval(Duration::from_millis(2));
/// for _ in 0..5 {
/// interval.tick().await;
@@ -174,6 +207,9 @@
/// # async fn main() {
/// let mut interval = interval(Duration::from_millis(50));
///
+ /// // First tick resolves immediately after creation
+ /// interval.tick().await;
+ ///
/// task_that_takes_200_millis().await;
/// // The `Interval` has missed a tick
///
@@ -242,7 +278,7 @@
/// // 50ms after the call to `tick` up above. That is, in `tick`, when we
/// // recognize that we missed a tick, we schedule the next tick to happen
/// // 50ms (or whatever the `period` is) from right then, not from when
- /// // were were *supposed* to tick
+ /// // were *supposed* to tick
/// interval.tick().await;
/// # }
/// ```
@@ -362,6 +398,9 @@
/// The strategy `Interval` should use when a tick is missed.
missed_tick_behavior: MissedTickBehavior,
+
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ resource_span: tracing::Span,
}
impl Interval {
@@ -384,6 +423,7 @@
/// let mut interval = time::interval(Duration::from_millis(10));
///
/// interval.tick().await;
+ /// // approximately 0ms have elapsed. The first tick completes immediately.
/// interval.tick().await;
/// interval.tick().await;
///
@@ -391,7 +431,20 @@
/// }
/// ```
pub async fn tick(&mut self) -> Instant {
- poll_fn(|cx| self.poll_tick(cx)).await
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let resource_span = self.resource_span.clone();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let instant = trace::async_op(
+ || poll_fn(|cx| self.poll_tick(cx)),
+ resource_span,
+ "Interval::tick",
+ "poll_tick",
+ false,
+ );
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
+ let instant = poll_fn(|cx| self.poll_tick(cx));
+
+ instant.await
}
/// Polls for the next instant in the interval to be reached.
@@ -435,6 +488,36 @@
Poll::Ready(timeout)
}
+ /// Resets the interval to complete one period after the current time.
+ ///
+ /// This method ignores [`MissedTickBehavior`] strategy.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// use tokio::time;
+ ///
+ /// use std::time::Duration;
+ ///
+ /// #[tokio::main]
+ /// async fn main() {
+ /// let mut interval = time::interval(Duration::from_millis(100));
+ ///
+ /// interval.tick().await;
+ ///
+ /// time::sleep(Duration::from_millis(50)).await;
+ /// interval.reset();
+ ///
+ /// interval.tick().await;
+ /// interval.tick().await;
+ ///
+ /// // approximately 250ms have elapsed.
+ /// }
+ /// ```
+ pub fn reset(&mut self) {
+ self.delay.as_mut().reset(Instant::now() + self.period);
+ }
+
/// Returns the [`MissedTickBehavior`] strategy currently being used.
pub fn missed_tick_behavior(&self) -> MissedTickBehavior {
self.missed_tick_behavior
diff --git a/src/time/mod.rs b/src/time/mod.rs
index 281990e..a1f27b8 100644
--- a/src/time/mod.rs
+++ b/src/time/mod.rs
@@ -82,17 +82,13 @@
//! ```
//!
//! [`interval`]: crate::time::interval()
+//! [`sleep`]: sleep()
mod clock;
pub(crate) use self::clock::Clock;
#[cfg(feature = "test-util")]
pub use clock::{advance, pause, resume};
-pub(crate) mod driver;
-
-#[doc(inline)]
-pub use driver::sleep::{sleep, sleep_until, Sleep};
-
pub mod error;
mod instant;
@@ -101,14 +97,13 @@
mod interval;
pub use interval::{interval, interval_at, Interval, MissedTickBehavior};
+mod sleep;
+pub use sleep::{sleep, sleep_until, Sleep};
+
mod timeout;
#[doc(inline)]
pub use timeout::{timeout, timeout_at, Timeout};
-#[cfg(test)]
-#[cfg(not(loom))]
-mod tests;
-
// Re-export for convenience
#[doc(no_inline)]
pub use std::time::Duration;
diff --git a/src/time/driver/sleep.rs b/src/time/sleep.rs
similarity index 69%
rename from src/time/driver/sleep.rs
rename to src/time/sleep.rs
index 43ff694..0a012e2 100644
--- a/src/time/driver/sleep.rs
+++ b/src/time/sleep.rs
@@ -1,4 +1,4 @@
-use crate::time::driver::{Handle, TimerEntry};
+use crate::runtime::time::TimerEntry;
use crate::time::{error::Error, Duration, Instant};
use crate::util::trace;
@@ -8,10 +8,6 @@
use std::pin::Pin;
use std::task::{self, Poll};
-cfg_trace! {
- use crate::time::driver::ClockTime;
-}
-
/// Waits until `deadline` is reached.
///
/// No work is performed while awaiting on the sleep future to complete. `Sleep`
@@ -41,11 +37,28 @@
///
/// See the documentation for the [`Sleep`] type for more examples.
///
+/// # Panics
+///
+/// This function panics if there is no current timer set.
+///
+/// It can be triggered when [`Builder::enable_time`] or
+/// [`Builder::enable_all`] are not included in the builder.
+///
+/// It can also panic whenever a timer is created outside of a
+/// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic,
+/// since the function is executed outside of the runtime.
+/// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic.
+/// And this is because wrapping the function on an async makes it lazy,
+/// and so gets executed inside the runtime successfully without
+/// panicking.
+///
/// [`Sleep`]: struct@crate::time::Sleep
/// [`interval`]: crate::time::interval()
+/// [`Builder::enable_time`]: crate::runtime::Builder::enable_time
+/// [`Builder::enable_all`]: crate::runtime::Builder::enable_all
// Alias for old name in 0.x
#[cfg_attr(docsrs, doc(alias = "delay_until"))]
-#[cfg_attr(tokio_track_caller, track_caller)]
+#[track_caller]
pub fn sleep_until(deadline: Instant) -> Sleep {
return Sleep::new_timeout(deadline, trace::caller_location());
}
@@ -57,7 +70,9 @@
///
/// No work is performed while awaiting on the sleep future to complete. `Sleep`
/// operates at millisecond granularity and should not be used for tasks that
-/// require high-resolution timers.
+/// require high-resolution timers. The implementation is platform specific,
+/// and some platforms (specifically Windows) will provide timers with a
+/// larger resolution than 1 ms.
///
/// To run something regularly on a schedule, see [`interval`].
///
@@ -84,12 +99,29 @@
///
/// See the documentation for the [`Sleep`] type for more examples.
///
+/// # Panics
+///
+/// This function panics if there is no current timer set.
+///
+/// It can be triggered when [`Builder::enable_time`] or
+/// [`Builder::enable_all`] are not included in the builder.
+///
+/// It can also panic whenever a timer is created outside of a
+/// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic,
+/// since the function is executed outside of the runtime.
+/// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic.
+/// And this is because wrapping the function on an async makes it lazy,
+/// and so gets executed inside the runtime successfully without
+/// panicking.
+///
/// [`Sleep`]: struct@crate::time::Sleep
/// [`interval`]: crate::time::interval()
+/// [`Builder::enable_time`]: crate::runtime::Builder::enable_time
+/// [`Builder::enable_all`]: crate::runtime::Builder::enable_all
// Alias for old name in 0.x
#[cfg_attr(docsrs, doc(alias = "delay_for"))]
#[cfg_attr(docsrs, doc(alias = "wait"))]
-#[cfg_attr(tokio_track_caller, track_caller)]
+#[track_caller]
pub fn sleep(duration: Duration) -> Sleep {
let location = trace::caller_location();
@@ -204,9 +236,7 @@
#[derive(Debug)]
struct Inner {
deadline: Instant,
- resource_span: tracing::Span,
- async_op_span: tracing::Span,
- time_source: ClockTime,
+ ctx: trace::AsyncOpTracingCtx,
}
}
@@ -219,23 +249,24 @@
impl Sleep {
#[cfg_attr(not(all(tokio_unstable, feature = "tracing")), allow(unused_variables))]
+ #[track_caller]
pub(crate) fn new_timeout(
deadline: Instant,
location: Option<&'static Location<'static>>,
) -> Sleep {
- let handle = Handle::current();
+ use crate::runtime::scheduler;
+
+ let handle = scheduler::Handle::current();
let entry = TimerEntry::new(&handle, deadline);
#[cfg(all(tokio_unstable, feature = "tracing"))]
let inner = {
- let time_source = handle.time_source().clone();
+ let handle = &handle.driver().time();
+ let time_source = handle.time_source();
let deadline_tick = time_source.deadline_to_tick(deadline);
- let duration = deadline_tick.checked_sub(time_source.now()).unwrap_or(0);
+ let duration = deadline_tick.saturating_sub(time_source.now());
- #[cfg(tokio_track_caller)]
- let location = location.expect("should have location if tracking caller");
-
- #[cfg(tokio_track_caller)]
+ let location = location.expect("should have location if tracing");
let resource_span = tracing::trace_span!(
"runtime.resource",
concrete_type = "Sleep",
@@ -245,27 +276,27 @@
loc.col = location.column(),
);
- #[cfg(not(tokio_track_caller))]
- let resource_span =
- tracing::trace_span!("runtime.resource", concrete_type = "Sleep", kind = "timer");
+ let async_op_span = resource_span.in_scope(|| {
+ tracing::trace!(
+ target: "runtime::resource::state_update",
+ duration = duration,
+ duration.unit = "ms",
+ duration.op = "override",
+ );
- let async_op_span =
- tracing::trace_span!("runtime.resource.async_op", source = "Sleep::new_timeout");
+ tracing::trace_span!("runtime.resource.async_op", source = "Sleep::new_timeout")
+ });
- tracing::trace!(
- target: "runtime::resource::state_update",
- parent: resource_span.id(),
- duration = duration,
- duration.unit = "ms",
- duration.op = "override",
- );
+ let async_op_poll_span =
+ async_op_span.in_scope(|| tracing::trace_span!("runtime.resource.async_op.poll"));
- Inner {
- deadline,
- resource_span,
+ let ctx = trace::AsyncOpTracingCtx {
async_op_span,
- time_source,
- }
+ async_op_poll_span,
+ resource_span,
+ };
+
+ Inner { deadline, ctx }
};
#[cfg(not(all(tokio_unstable, feature = "tracing")))]
@@ -324,60 +355,59 @@
}
fn reset_inner(self: Pin<&mut Self>, deadline: Instant) {
- let me = self.project();
- me.entry.reset(deadline);
- (*me.inner).deadline = deadline;
+ let mut me = self.project();
+ me.entry.as_mut().reset(deadline);
+ (me.inner).deadline = deadline;
#[cfg(all(tokio_unstable, feature = "tracing"))]
{
- me.inner.async_op_span =
+ let _resource_enter = me.inner.ctx.resource_span.enter();
+ me.inner.ctx.async_op_span =
tracing::trace_span!("runtime.resource.async_op", source = "Sleep::reset");
+ let _async_op_enter = me.inner.ctx.async_op_span.enter();
+
+ me.inner.ctx.async_op_poll_span =
+ tracing::trace_span!("runtime.resource.async_op.poll");
+
+ let duration = {
+ let time_source = me.entry.driver().time_source();
+ let now = time_source.now();
+ let deadline_tick = time_source.deadline_to_tick(deadline);
+ deadline_tick.saturating_sub(now)
+ };
tracing::trace!(
target: "runtime::resource::state_update",
- parent: me.inner.resource_span.id(),
- duration = {
- let now = me.inner.time_source.now();
- let deadline_tick = me.inner.time_source.deadline_to_tick(deadline);
- deadline_tick.checked_sub(now).unwrap_or(0)
- },
+ duration = duration,
duration.unit = "ms",
duration.op = "override",
);
}
}
- cfg_not_trace! {
- fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> {
- let me = self.project();
+ fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> {
+ let me = self.project();
- // Keep track of task budget
- let coop = ready!(crate::coop::poll_proceed(cx));
+ // Keep track of task budget
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let coop = ready!(trace_poll_op!(
+ "poll_elapsed",
+ crate::runtime::coop::poll_proceed(cx),
+ ));
- me.entry.poll_elapsed(cx).map(move |r| {
- coop.made_progress();
- r
- })
- }
- }
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ let coop = ready!(crate::runtime::coop::poll_proceed(cx));
- cfg_trace! {
- fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> {
- let me = self.project();
- // Keep track of task budget
- let coop = ready!(trace_poll_op!(
- "poll_elapsed",
- crate::coop::poll_proceed(cx),
- me.inner.resource_span.id(),
- ));
+ let result = me.entry.poll_elapsed(cx).map(move |r| {
+ coop.made_progress();
+ r
+ });
- let result = me.entry.poll_elapsed(cx).map(move |r| {
- coop.made_progress();
- r
- });
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ return trace_poll_op!("poll_elapsed", result);
- trace_poll_op!("poll_elapsed", result, me.inner.resource_span.id())
- }
+ #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
+ return result;
}
}
@@ -395,8 +425,11 @@
// really do much better if we passed the error onwards.
fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> {
#[cfg(all(tokio_unstable, feature = "tracing"))]
- let _span = self.inner.async_op_span.clone().entered();
-
+ let _res_span = self.inner.ctx.resource_span.clone().entered();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _ao_span = self.inner.ctx.async_op_span.clone().entered();
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
+ let _ao_poll_span = self.inner.ctx.async_op_poll_span.clone().entered();
match ready!(self.as_mut().poll_elapsed(cx)) {
Ok(()) => Poll::Ready(()),
Err(e) => panic!("timer error: {}", e),
diff --git a/src/time/tests/mod.rs b/src/time/tests/mod.rs
deleted file mode 100644
index 35e1060..0000000
--- a/src/time/tests/mod.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-mod test_sleep;
-
-use crate::time::{self, Instant};
-use std::time::Duration;
-
-fn assert_send<T: Send>() {}
-fn assert_sync<T: Sync>() {}
-
-#[test]
-fn registration_is_send_and_sync() {
- use crate::time::Sleep;
-
- assert_send::<Sleep>();
- assert_sync::<Sleep>();
-}
-
-#[test]
-#[should_panic]
-fn sleep_is_eager() {
- let when = Instant::now() + Duration::from_millis(100);
- let _ = time::sleep_until(when);
-}
diff --git a/src/time/tests/test_sleep.rs b/src/time/tests/test_sleep.rs
deleted file mode 100644
index 77ca07e..0000000
--- a/src/time/tests/test_sleep.rs
+++ /dev/null
@@ -1,443 +0,0 @@
-//use crate::time::driver::{Driver, Entry, Handle};
-
-/*
-macro_rules! poll {
- ($e:expr) => {
- $e.enter(|cx, e| e.poll_elapsed(cx))
- };
-}
-
-#[test]
-fn frozen_utility_returns_correct_advanced_duration() {
- let clock = Clock::new();
- clock.pause();
- let start = clock.now();
-
- clock.advance(ms(10));
- assert_eq!(clock.now() - start, ms(10));
-}
-
-#[test]
-fn immediate_sleep() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- let when = clock.now();
- let mut e = task::spawn(sleep_until(&handle, when));
-
- assert_ready_ok!(poll!(e));
-
- assert_ok!(driver.park_timeout(Duration::from_millis(1000)));
-
- // The time has not advanced. The `turn` completed immediately.
- assert_eq!(clock.now() - start, ms(1000));
-}
-
-#[test]
-fn delayed_sleep_level_0() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- for &i in &[1, 10, 60] {
- // Create a `Sleep` that elapses in the future
- let mut e = task::spawn(sleep_until(&handle, start + ms(i)));
-
- // The sleep instance has not elapsed.
- assert_pending!(poll!(e));
-
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(i));
-
- assert_ready_ok!(poll!(e));
- }
-}
-
-#[test]
-fn sub_ms_delayed_sleep() {
- let (mut driver, clock, handle) = setup();
-
- for _ in 0..5 {
- let deadline = clock.now() + ms(1) + Duration::new(0, 1);
-
- let mut e = task::spawn(sleep_until(&handle, deadline));
-
- assert_pending!(poll!(e));
-
- assert_ok!(driver.park());
- assert_ready_ok!(poll!(e));
-
- assert!(clock.now() >= deadline);
-
- clock.advance(Duration::new(0, 1));
- }
-}
-
-#[test]
-fn delayed_sleep_wrapping_level_0() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- assert_ok!(driver.park_timeout(ms(5)));
- assert_eq!(clock.now() - start, ms(5));
-
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(60)));
-
- assert_pending!(poll!(e));
-
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(64));
- assert_pending!(poll!(e));
-
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(65));
-
- assert_ready_ok!(poll!(e));
-}
-
-#[test]
-fn timer_wrapping_with_higher_levels() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Set sleep to hit level 1
- let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(64)));
- assert_pending!(poll!(e1));
-
- // Turn a bit
- assert_ok!(driver.park_timeout(ms(5)));
-
- // Set timeout such that it will hit level 0, but wrap
- let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(60)));
- assert_pending!(poll!(e2));
-
- // This should result in s1 firing
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(64));
-
- assert_ready_ok!(poll!(e1));
- assert_pending!(poll!(e2));
-
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(65));
-
- assert_ready_ok!(poll!(e1));
-}
-
-#[test]
-fn sleep_with_deadline_in_past() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Create `Sleep` that elapsed immediately.
- let mut e = task::spawn(sleep_until(&handle, clock.now() - ms(100)));
-
- // Even though the `Sleep` expires in the past, it is not ready yet
- // because the timer must observe it.
- assert_ready_ok!(poll!(e));
-
- // Turn the timer, it runs for the elapsed time
- assert_ok!(driver.park_timeout(ms(1000)));
-
- // The time has not advanced. The `turn` completed immediately.
- assert_eq!(clock.now() - start, ms(1000));
-}
-
-#[test]
-fn delayed_sleep_level_1() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Create a `Sleep` that elapses in the future
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(234)));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- // Turn the timer, this will wake up to cascade the timer down.
- assert_ok!(driver.park_timeout(ms(1000)));
- assert_eq!(clock.now() - start, ms(192));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- // Turn the timer again
- assert_ok!(driver.park_timeout(ms(1000)));
- assert_eq!(clock.now() - start, ms(234));
-
- // The sleep has elapsed.
- assert_ready_ok!(poll!(e));
-
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Create a `Sleep` that elapses in the future
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(234)));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- // Turn the timer with a smaller timeout than the cascade.
- assert_ok!(driver.park_timeout(ms(100)));
- assert_eq!(clock.now() - start, ms(100));
-
- assert_pending!(poll!(e));
-
- // Turn the timer, this will wake up to cascade the timer down.
- assert_ok!(driver.park_timeout(ms(1000)));
- assert_eq!(clock.now() - start, ms(192));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- // Turn the timer again
- assert_ok!(driver.park_timeout(ms(1000)));
- assert_eq!(clock.now() - start, ms(234));
-
- // The sleep has elapsed.
- assert_ready_ok!(poll!(e));
-}
-
-#[test]
-fn concurrently_set_two_timers_second_one_shorter() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(500)));
- let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(200)));
-
- // The sleep has not elapsed
- assert_pending!(poll!(e1));
- assert_pending!(poll!(e2));
-
- // Sleep until a cascade
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(192));
-
- // Sleep until the second timer.
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(200));
-
- // The shorter sleep fires
- assert_ready_ok!(poll!(e2));
- assert_pending!(poll!(e1));
-
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(448));
-
- assert_pending!(poll!(e1));
-
- // Turn again, this time the time will advance to the second sleep
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(500));
-
- assert_ready_ok!(poll!(e1));
-}
-
-#[test]
-fn short_sleep() {
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Create a `Sleep` that elapses in the future
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(1)));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- // Turn the timer, but not enough time will go by.
- assert_ok!(driver.park());
-
- // The sleep has elapsed.
- assert_ready_ok!(poll!(e));
-
- // The time has advanced to the point of the sleep elapsing.
- assert_eq!(clock.now() - start, ms(1));
-}
-
-#[test]
-fn sorta_long_sleep_until() {
- const MIN_5: u64 = 5 * 60 * 1000;
-
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Create a `Sleep` that elapses in the future
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(MIN_5)));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- let cascades = &[262_144, 262_144 + 9 * 4096, 262_144 + 9 * 4096 + 15 * 64];
-
- for &elapsed in cascades {
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(elapsed));
-
- assert_pending!(poll!(e));
- }
-
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(MIN_5));
-
- // The sleep has elapsed.
- assert_ready_ok!(poll!(e));
-}
-
-#[test]
-fn very_long_sleep() {
- const MO_5: u64 = 5 * 30 * 24 * 60 * 60 * 1000;
-
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- // Create a `Sleep` that elapses in the future
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(MO_5)));
-
- // The sleep has not elapsed.
- assert_pending!(poll!(e));
-
- let cascades = &[
- 12_884_901_888,
- 12_952_010_752,
- 12_959_875_072,
- 12_959_997_952,
- ];
-
- for &elapsed in cascades {
- assert_ok!(driver.park());
- assert_eq!(clock.now() - start, ms(elapsed));
-
- assert_pending!(poll!(e));
- }
-
- // Turn the timer, but not enough time will go by.
- assert_ok!(driver.park());
-
- // The time has advanced to the point of the sleep elapsing.
- assert_eq!(clock.now() - start, ms(MO_5));
-
- // The sleep has elapsed.
- assert_ready_ok!(poll!(e));
-}
-
-#[test]
-fn unpark_is_delayed() {
- // A special park that will take much longer than the requested duration
- struct MockPark(Clock);
-
- struct MockUnpark;
-
- impl Park for MockPark {
- type Unpark = MockUnpark;
- type Error = ();
-
- fn unpark(&self) -> Self::Unpark {
- MockUnpark
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- panic!("parking forever");
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- assert_eq!(duration, ms(0));
- self.0.advance(ms(436));
- Ok(())
- }
-
- fn shutdown(&mut self) {}
- }
-
- impl Unpark for MockUnpark {
- fn unpark(&self) {}
- }
-
- let clock = Clock::new();
- clock.pause();
- let start = clock.now();
- let mut driver = Driver::new(MockPark(clock.clone()), clock.clone());
- let handle = driver.handle();
-
- let mut e1 = task::spawn(sleep_until(&handle, clock.now() + ms(100)));
- let mut e2 = task::spawn(sleep_until(&handle, clock.now() + ms(101)));
- let mut e3 = task::spawn(sleep_until(&handle, clock.now() + ms(200)));
-
- assert_pending!(poll!(e1));
- assert_pending!(poll!(e2));
- assert_pending!(poll!(e3));
-
- assert_ok!(driver.park());
-
- assert_eq!(clock.now() - start, ms(500));
-
- assert_ready_ok!(poll!(e1));
- assert_ready_ok!(poll!(e2));
- assert_ready_ok!(poll!(e3));
-}
-
-#[test]
-fn set_timeout_at_deadline_greater_than_max_timer() {
- const YR_1: u64 = 365 * 24 * 60 * 60 * 1000;
- const YR_5: u64 = 5 * YR_1;
-
- let (mut driver, clock, handle) = setup();
- let start = clock.now();
-
- for _ in 0..5 {
- assert_ok!(driver.park_timeout(ms(YR_1)));
- }
-
- let mut e = task::spawn(sleep_until(&handle, clock.now() + ms(1)));
- assert_pending!(poll!(e));
-
- assert_ok!(driver.park_timeout(ms(1000)));
- assert_eq!(clock.now() - start, ms(YR_5) + ms(1));
-
- assert_ready_ok!(poll!(e));
-}
-
-fn setup() -> (Driver<MockPark>, Clock, Handle) {
- let clock = Clock::new();
- clock.pause();
- let driver = Driver::new(MockPark(clock.clone()), clock.clone());
- let handle = driver.handle();
-
- (driver, clock, handle)
-}
-
-fn sleep_until(handle: &Handle, when: Instant) -> Arc<Entry> {
- Entry::new(&handle, when, ms(0))
-}
-
-struct MockPark(Clock);
-
-struct MockUnpark;
-
-impl Park for MockPark {
- type Unpark = MockUnpark;
- type Error = ();
-
- fn unpark(&self) -> Self::Unpark {
- MockUnpark
- }
-
- fn park(&mut self) -> Result<(), Self::Error> {
- panic!("parking forever");
- }
-
- fn park_timeout(&mut self, duration: Duration) -> Result<(), Self::Error> {
- self.0.advance(duration);
- Ok(())
- }
-
- fn shutdown(&mut self) {}
-}
-
-impl Unpark for MockUnpark {
- fn unpark(&self) {}
-}
-
-fn ms(n: u64) -> Duration {
- Duration::from_millis(n)
-}
-*/
diff --git a/src/time/timeout.rs b/src/time/timeout.rs
index 6725caa..3bb98ea 100644
--- a/src/time/timeout.rs
+++ b/src/time/timeout.rs
@@ -5,6 +5,7 @@
//! [`Timeout`]: struct@Timeout
use crate::{
+ runtime::coop,
time::{error::Elapsed, sleep_until, Duration, Instant, Sleep},
util::trace,
};
@@ -20,7 +21,17 @@
/// value is returned. Otherwise, an error is returned and the future is
/// canceled.
///
-/// # Cancelation
+/// Note that the timeout is checked before polling the future, so if the future
+/// does not yield during execution then it is possible for the future to complete
+/// and exceed the timeout _without_ returning an error.
+///
+/// This function returns a future whose return type is [`Result`]`<T,`[`Elapsed`]`>`, where `T` is the
+/// return type of the provided future.
+///
+/// [`Result`]: std::result::Result
+/// [`Elapsed`]: crate::time::error::Elapsed
+///
+/// # Cancellation
///
/// Cancelling a timeout is done by dropping the future. No additional cleanup
/// or other work is required.
@@ -48,10 +59,28 @@
/// }
/// # }
/// ```
-#[cfg_attr(tokio_track_caller, track_caller)]
-pub fn timeout<T>(duration: Duration, future: T) -> Timeout<T>
+///
+/// # Panics
+///
+/// This function panics if there is no current timer set.
+///
+/// It can be triggered when [`Builder::enable_time`] or
+/// [`Builder::enable_all`] are not included in the builder.
+///
+/// It can also panic whenever a timer is created outside of a
+/// Tokio runtime. That is why `rt.block_on(sleep(...))` will panic,
+/// since the function is executed outside of the runtime.
+/// Whereas `rt.block_on(async {sleep(...).await})` doesn't panic.
+/// And this is because wrapping the function on an async makes it lazy,
+/// and so gets executed inside the runtime successfully without
+/// panicking.
+///
+/// [`Builder::enable_time`]: crate::runtime::Builder::enable_time
+/// [`Builder::enable_all`]: crate::runtime::Builder::enable_all
+#[track_caller]
+pub fn timeout<F>(duration: Duration, future: F) -> Timeout<F>
where
- T: Future,
+ F: Future,
{
let location = trace::caller_location();
@@ -68,7 +97,13 @@
/// If the future completes before the instant is reached, then the completed
/// value is returned. Otherwise, an error is returned.
///
-/// # Cancelation
+/// This function returns a future whose return type is [`Result`]`<T,`[`Elapsed`]`>`, where `T` is the
+/// return type of the provided future.
+///
+/// [`Result`]: std::result::Result
+/// [`Elapsed`]: crate::time::error::Elapsed
+///
+/// # Cancellation
///
/// Cancelling a timeout is done by dropping the future. No additional cleanup
/// or other work is required.
@@ -97,9 +132,9 @@
/// }
/// # }
/// ```
-pub fn timeout_at<T>(deadline: Instant, future: T) -> Timeout<T>
+pub fn timeout_at<F>(deadline: Instant, future: F) -> Timeout<F>
where
- T: Future,
+ F: Future,
{
let delay = sleep_until(deadline);
@@ -151,15 +186,33 @@
fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> {
let me = self.project();
+ let had_budget_before = coop::has_budget_remaining();
+
// First, try polling the future
if let Poll::Ready(v) = me.value.poll(cx) {
return Poll::Ready(Ok(v));
}
- // Now check the timer
- match me.delay.poll(cx) {
- Poll::Ready(()) => Poll::Ready(Err(Elapsed::new())),
- Poll::Pending => Poll::Pending,
+ let has_budget_now = coop::has_budget_remaining();
+
+ let delay = me.delay;
+
+ let poll_delay = || -> Poll<Self::Output> {
+ match delay.poll(cx) {
+ Poll::Ready(()) => Poll::Ready(Err(Elapsed::new())),
+ Poll::Pending => Poll::Pending,
+ }
+ };
+
+ if let (true, false) = (had_budget_before, has_budget_now) {
+ // if it is the underlying future that exhausted the budget, we poll
+ // the `delay` with an unconstrained one. This prevents pathological
+ // cases where the underlying future always exhausts the budget and
+ // we never get a chance to evaluate whether the timeout was hit or
+ // not.
+ coop::with_unconstrained(poll_delay)
+ } else {
+ poll_delay()
}
}
}
diff --git a/src/runtime/thread_pool/atomic_cell.rs b/src/util/atomic_cell.rs
similarity index 77%
rename from src/runtime/thread_pool/atomic_cell.rs
rename to src/util/atomic_cell.rs
index 98847e6..07e3730 100644
--- a/src/runtime/thread_pool/atomic_cell.rs
+++ b/src/util/atomic_cell.rs
@@ -3,7 +3,7 @@
use std::ptr;
use std::sync::atomic::Ordering::AcqRel;
-pub(super) struct AtomicCell<T> {
+pub(crate) struct AtomicCell<T> {
data: AtomicPtr<T>,
}
@@ -11,22 +11,22 @@
unsafe impl<T: Send> Sync for AtomicCell<T> {}
impl<T> AtomicCell<T> {
- pub(super) fn new(data: Option<Box<T>>) -> AtomicCell<T> {
+ pub(crate) fn new(data: Option<Box<T>>) -> AtomicCell<T> {
AtomicCell {
data: AtomicPtr::new(to_raw(data)),
}
}
- pub(super) fn swap(&self, val: Option<Box<T>>) -> Option<Box<T>> {
+ pub(crate) fn swap(&self, val: Option<Box<T>>) -> Option<Box<T>> {
let old = self.data.swap(to_raw(val), AcqRel);
from_raw(old)
}
- pub(super) fn set(&self, val: Box<T>) {
+ pub(crate) fn set(&self, val: Box<T>) {
let _ = self.swap(Some(val));
}
- pub(super) fn take(&self) -> Option<Box<T>> {
+ pub(crate) fn take(&self) -> Option<Box<T>> {
self.swap(None)
}
}
diff --git a/src/util/error.rs b/src/util/error.rs
index 8f252c0..ebb27f6 100644
--- a/src/util/error.rs
+++ b/src/util/error.rs
@@ -1,15 +1,14 @@
+// Some combinations of features may not use these constants.
+#![cfg_attr(not(feature = "full"), allow(dead_code))]
+
/// Error string explaining that the Tokio context hasn't been instantiated.
pub(crate) const CONTEXT_MISSING_ERROR: &str =
"there is no reactor running, must be called from the context of a Tokio 1.x runtime";
-// some combinations of features might not use this
-#[allow(dead_code)]
/// Error string explaining that the Tokio context is shutting down and cannot drive timers.
pub(crate) const RUNTIME_SHUTTING_DOWN_ERROR: &str =
"A Tokio 1.x context was found, but it is being shutdown.";
-// some combinations of features might not use this
-#[allow(dead_code)]
/// Error string explaining that the Tokio context is not available because the
/// thread-local storing it has been destroyed. This usually only happens during
/// destructors of other thread-locals.
diff --git a/src/util/idle_notified_set.rs b/src/util/idle_notified_set.rs
new file mode 100644
index 0000000..ce8ff9e
--- /dev/null
+++ b/src/util/idle_notified_set.rs
@@ -0,0 +1,462 @@
+//! This module defines an `IdleNotifiedSet`, which is a collection of elements.
+//! Each element is intended to correspond to a task, and the collection will
+//! keep track of which tasks have had their waker notified, and which have not.
+//!
+//! Each entry in the set holds some user-specified value. The value's type is
+//! specified using the `T` parameter. It will usually be a `JoinHandle` or
+//! similar.
+
+use std::marker::PhantomPinned;
+use std::mem::ManuallyDrop;
+use std::ptr::NonNull;
+use std::task::{Context, Waker};
+
+use crate::loom::cell::UnsafeCell;
+use crate::loom::sync::{Arc, Mutex};
+use crate::util::linked_list::{self, Link};
+use crate::util::{waker_ref, Wake};
+
+type LinkedList<T> =
+ linked_list::LinkedList<ListEntry<T>, <ListEntry<T> as linked_list::Link>::Target>;
+
+/// This is the main handle to the collection.
+pub(crate) struct IdleNotifiedSet<T> {
+ lists: Arc<Lists<T>>,
+ length: usize,
+}
+
+/// A handle to an entry that is guaranteed to be stored in the idle or notified
+/// list of its `IdleNotifiedSet`. This value borrows the `IdleNotifiedSet`
+/// mutably to prevent the entry from being moved to the `Neither` list, which
+/// only the `IdleNotifiedSet` may do.
+///
+/// The main consequence of being stored in one of the lists is that the `value`
+/// field has not yet been consumed.
+///
+/// Note: This entry can be moved from the idle to the notified list while this
+/// object exists by waking its waker.
+pub(crate) struct EntryInOneOfTheLists<'a, T> {
+ entry: Arc<ListEntry<T>>,
+ set: &'a mut IdleNotifiedSet<T>,
+}
+
+type Lists<T> = Mutex<ListsInner<T>>;
+
+/// The linked lists hold strong references to the ListEntry items, and the
+/// ListEntry items also hold a strong reference back to the Lists object, but
+/// the destructor of the `IdleNotifiedSet` will clear the two lists, so once
+/// that object is destroyed, no ref-cycles will remain.
+struct ListsInner<T> {
+ notified: LinkedList<T>,
+ idle: LinkedList<T>,
+ /// Whenever an element in the `notified` list is woken, this waker will be
+ /// notified and consumed, if it exists.
+ waker: Option<Waker>,
+}
+
+/// Which of the two lists in the shared Lists object is this entry stored in?
+///
+/// If the value is `Idle`, then an entry's waker may move it to the notified
+/// list. Otherwise, only the `IdleNotifiedSet` may move it.
+///
+/// If the value is `Neither`, then it is still possible that the entry is in
+/// some third external list (this happens in `drain`).
+#[derive(Copy, Clone, Eq, PartialEq)]
+enum List {
+ Notified,
+ Idle,
+ Neither,
+}
+
+/// An entry in the list.
+///
+/// # Safety
+///
+/// The `my_list` field must only be accessed while holding the mutex in
+/// `parent`. It is an invariant that the value of `my_list` corresponds to
+/// which linked list in the `parent` holds this entry. Once this field takes
+/// the value `Neither`, then it may never be modified again.
+///
+/// If the value of `my_list` is `Notified` or `Idle`, then the `pointers` field
+/// must only be accessed while holding the mutex. If the value of `my_list` is
+/// `Neither`, then the `pointers` field may be accessed by the
+/// `IdleNotifiedSet` (this happens inside `drain`).
+///
+/// The `value` field is owned by the `IdleNotifiedSet` and may only be accessed
+/// by the `IdleNotifiedSet`. The operation that sets the value of `my_list` to
+/// `Neither` assumes ownership of the `value`, and it must either drop it or
+/// move it out from this entry to prevent it from getting leaked. (Since the
+/// two linked lists are emptied in the destructor of `IdleNotifiedSet`, the
+/// value should not be leaked.)
+struct ListEntry<T> {
+ /// The linked list pointers of the list this entry is in.
+ pointers: linked_list::Pointers<ListEntry<T>>,
+ /// Pointer to the shared `Lists` struct.
+ parent: Arc<Lists<T>>,
+ /// The value stored in this entry.
+ value: UnsafeCell<ManuallyDrop<T>>,
+ /// Used to remember which list this entry is in.
+ my_list: UnsafeCell<List>,
+ /// Required by the `linked_list::Pointers` field.
+ _pin: PhantomPinned,
+}
+
+generate_addr_of_methods! {
+ impl<T> ListEntry<T> {
+ unsafe fn addr_of_pointers(self: NonNull<Self>) -> NonNull<linked_list::Pointers<ListEntry<T>>> {
+ &self.pointers
+ }
+ }
+}
+
+// With mutable access to the `IdleNotifiedSet`, you can get mutable access to
+// the values.
+unsafe impl<T: Send> Send for IdleNotifiedSet<T> {}
+// With the current API we strictly speaking don't even need `T: Sync`, but we
+// require it anyway to support adding &self APIs that access the values in the
+// future.
+unsafe impl<T: Sync> Sync for IdleNotifiedSet<T> {}
+
+// These impls control when it is safe to create a Waker. Since the waker does
+// not allow access to the value in any way (including its destructor), it is
+// not necessary for `T` to be Send or Sync.
+unsafe impl<T> Send for ListEntry<T> {}
+unsafe impl<T> Sync for ListEntry<T> {}
+
+impl<T> IdleNotifiedSet<T> {
+ /// Create a new IdleNotifiedSet.
+ pub(crate) fn new() -> Self {
+ let lists = Mutex::new(ListsInner {
+ notified: LinkedList::new(),
+ idle: LinkedList::new(),
+ waker: None,
+ });
+
+ IdleNotifiedSet {
+ lists: Arc::new(lists),
+ length: 0,
+ }
+ }
+
+ pub(crate) fn len(&self) -> usize {
+ self.length
+ }
+
+ pub(crate) fn is_empty(&self) -> bool {
+ self.length == 0
+ }
+
+ /// Insert the given value into the `idle` list.
+ pub(crate) fn insert_idle(&mut self, value: T) -> EntryInOneOfTheLists<'_, T> {
+ self.length += 1;
+
+ let entry = Arc::new(ListEntry {
+ parent: self.lists.clone(),
+ value: UnsafeCell::new(ManuallyDrop::new(value)),
+ my_list: UnsafeCell::new(List::Idle),
+ pointers: linked_list::Pointers::new(),
+ _pin: PhantomPinned,
+ });
+
+ {
+ let mut lock = self.lists.lock();
+ lock.idle.push_front(entry.clone());
+ }
+
+ // Safety: We just put the entry in the idle list, so it is in one of the lists.
+ EntryInOneOfTheLists { entry, set: self }
+ }
+
+ /// Pop an entry from the notified list to poll it. The entry is moved to
+ /// the idle list atomically.
+ pub(crate) fn pop_notified(&mut self, waker: &Waker) -> Option<EntryInOneOfTheLists<'_, T>> {
+ // We don't decrement the length because this call moves the entry to
+ // the idle list rather than removing it.
+ if self.length == 0 {
+ // Fast path.
+ return None;
+ }
+
+ let mut lock = self.lists.lock();
+
+ let should_update_waker = match lock.waker.as_mut() {
+ Some(cur_waker) => !waker.will_wake(cur_waker),
+ None => true,
+ };
+ if should_update_waker {
+ lock.waker = Some(waker.clone());
+ }
+
+ // Pop the entry, returning None if empty.
+ let entry = lock.notified.pop_back()?;
+
+ lock.idle.push_front(entry.clone());
+
+ // Safety: We are holding the lock.
+ entry.my_list.with_mut(|ptr| unsafe {
+ *ptr = List::Idle;
+ });
+
+ drop(lock);
+
+ // Safety: We just put the entry in the idle list, so it is in one of the lists.
+ Some(EntryInOneOfTheLists { entry, set: self })
+ }
+
+ /// Call a function on every element in this list.
+ pub(crate) fn for_each<F: FnMut(&mut T)>(&mut self, mut func: F) {
+ fn get_ptrs<T>(list: &mut LinkedList<T>, ptrs: &mut Vec<*mut T>) {
+ let mut node = list.last();
+
+ while let Some(entry) = node {
+ ptrs.push(entry.value.with_mut(|ptr| {
+ let ptr: *mut ManuallyDrop<T> = ptr;
+ let ptr: *mut T = ptr.cast();
+ ptr
+ }));
+
+ let prev = entry.pointers.get_prev();
+ node = prev.map(|prev| unsafe { &*prev.as_ptr() });
+ }
+ }
+
+ // Atomically get a raw pointer to the value of every entry.
+ //
+ // Since this only locks the mutex once, it is not possible for a value
+ // to get moved from the idle list to the notified list during the
+ // operation, which would otherwise result in some value being listed
+ // twice.
+ let mut ptrs = Vec::with_capacity(self.len());
+ {
+ let mut lock = self.lists.lock();
+
+ get_ptrs(&mut lock.idle, &mut ptrs);
+ get_ptrs(&mut lock.notified, &mut ptrs);
+ }
+ debug_assert_eq!(ptrs.len(), ptrs.capacity());
+
+ for ptr in ptrs {
+ // Safety: When we grabbed the pointers, the entries were in one of
+ // the two lists. This means that their value was valid at the time,
+ // and it must still be valid because we are the IdleNotifiedSet,
+ // and only we can remove an entry from the two lists. (It's
+ // possible that an entry is moved from one list to the other during
+ // this loop, but that is ok.)
+ func(unsafe { &mut *ptr });
+ }
+ }
+
+ /// Remove all entries in both lists, applying some function to each element.
+ ///
+ /// The closure is called on all elements even if it panics. Having it panic
+ /// twice is a double-panic, and will abort the application.
+ pub(crate) fn drain<F: FnMut(T)>(&mut self, func: F) {
+ if self.length == 0 {
+ // Fast path.
+ return;
+ }
+ self.length = 0;
+
+ // The LinkedList is not cleared on panic, so we use a bomb to clear it.
+ //
+ // This value has the invariant that any entry in its `all_entries` list
+ // has `my_list` set to `Neither` and that the value has not yet been
+ // dropped.
+ struct AllEntries<T, F: FnMut(T)> {
+ all_entries: LinkedList<T>,
+ func: F,
+ }
+
+ impl<T, F: FnMut(T)> AllEntries<T, F> {
+ fn pop_next(&mut self) -> bool {
+ if let Some(entry) = self.all_entries.pop_back() {
+ // Safety: We just took this value from the list, so we can
+ // destroy the value in the entry.
+ entry
+ .value
+ .with_mut(|ptr| unsafe { (self.func)(ManuallyDrop::take(&mut *ptr)) });
+ true
+ } else {
+ false
+ }
+ }
+ }
+
+ impl<T, F: FnMut(T)> Drop for AllEntries<T, F> {
+ fn drop(&mut self) {
+ while self.pop_next() {}
+ }
+ }
+
+ let mut all_entries = AllEntries {
+ all_entries: LinkedList::new(),
+ func,
+ };
+
+ // Atomically move all entries to the new linked list in the AllEntries
+ // object.
+ {
+ let mut lock = self.lists.lock();
+ unsafe {
+ // Safety: We are holding the lock and `all_entries` is a new
+ // LinkedList.
+ move_to_new_list(&mut lock.idle, &mut all_entries.all_entries);
+ move_to_new_list(&mut lock.notified, &mut all_entries.all_entries);
+ }
+ }
+
+ // Keep destroying entries in the list until it is empty.
+ //
+ // If the closure panics, then the destructor of the `AllEntries` bomb
+ // ensures that we keep running the destructor on the remaining values.
+ // A second panic will abort the program.
+ while all_entries.pop_next() {}
+ }
+}
+
+/// # Safety
+///
+/// The mutex for the entries must be held, and the target list must be such
+/// that setting `my_list` to `Neither` is ok.
+unsafe fn move_to_new_list<T>(from: &mut LinkedList<T>, to: &mut LinkedList<T>) {
+ while let Some(entry) = from.pop_back() {
+ entry.my_list.with_mut(|ptr| {
+ *ptr = List::Neither;
+ });
+ to.push_front(entry);
+ }
+}
+
+impl<'a, T> EntryInOneOfTheLists<'a, T> {
+ /// Remove this entry from the list it is in, returning the value associated
+ /// with the entry.
+ ///
+ /// This consumes the value, since it is no longer guaranteed to be in a
+ /// list.
+ pub(crate) fn remove(self) -> T {
+ self.set.length -= 1;
+
+ {
+ let mut lock = self.set.lists.lock();
+
+ // Safety: We are holding the lock so there is no race, and we will
+ // remove the entry afterwards to uphold invariants.
+ let old_my_list = self.entry.my_list.with_mut(|ptr| unsafe {
+ let old_my_list = *ptr;
+ *ptr = List::Neither;
+ old_my_list
+ });
+
+ let list = match old_my_list {
+ List::Idle => &mut lock.idle,
+ List::Notified => &mut lock.notified,
+ // An entry in one of the lists is in one of the lists.
+ List::Neither => unreachable!(),
+ };
+
+ unsafe {
+ // Safety: We just checked that the entry is in this particular
+ // list.
+ list.remove(ListEntry::as_raw(&self.entry)).unwrap();
+ }
+ }
+
+ // By setting `my_list` to `Neither`, we have taken ownership of the
+ // value. We return it to the caller.
+ //
+ // Safety: We have a mutable reference to the `IdleNotifiedSet` that
+ // owns this entry, so we can use its permission to access the value.
+ self.entry
+ .value
+ .with_mut(|ptr| unsafe { ManuallyDrop::take(&mut *ptr) })
+ }
+
+ /// Access the value in this entry together with a context for its waker.
+ pub(crate) fn with_value_and_context<F, U>(&mut self, func: F) -> U
+ where
+ F: FnOnce(&mut T, &mut Context<'_>) -> U,
+ T: 'static,
+ {
+ let waker = waker_ref(&self.entry);
+
+ let mut context = Context::from_waker(&waker);
+
+ // Safety: We have a mutable reference to the `IdleNotifiedSet` that
+ // owns this entry, so we can use its permission to access the value.
+ self.entry
+ .value
+ .with_mut(|ptr| unsafe { func(&mut *ptr, &mut context) })
+ }
+}
+
+impl<T> Drop for IdleNotifiedSet<T> {
+ fn drop(&mut self) {
+ // Clear both lists.
+ self.drain(drop);
+
+ #[cfg(debug_assertions)]
+ if !std::thread::panicking() {
+ let lock = self.lists.lock();
+ assert!(lock.idle.is_empty());
+ assert!(lock.notified.is_empty());
+ }
+ }
+}
+
+impl<T: 'static> Wake for ListEntry<T> {
+ fn wake_by_ref(me: &Arc<Self>) {
+ let mut lock = me.parent.lock();
+
+ // Safety: We are holding the lock and we will update the lists to
+ // maintain invariants.
+ let old_my_list = me.my_list.with_mut(|ptr| unsafe {
+ let old_my_list = *ptr;
+ if old_my_list == List::Idle {
+ *ptr = List::Notified;
+ }
+ old_my_list
+ });
+
+ if old_my_list == List::Idle {
+ // We move ourself to the notified list.
+ let me = unsafe {
+ // Safety: We just checked that we are in this particular list.
+ lock.idle.remove(NonNull::from(&**me)).unwrap()
+ };
+ lock.notified.push_front(me);
+
+ if let Some(waker) = lock.waker.take() {
+ drop(lock);
+ waker.wake();
+ }
+ }
+ }
+
+ fn wake(me: Arc<Self>) {
+ Self::wake_by_ref(&me)
+ }
+}
+
+/// # Safety
+///
+/// `ListEntry` is forced to be !Unpin.
+unsafe impl<T> linked_list::Link for ListEntry<T> {
+ type Handle = Arc<ListEntry<T>>;
+ type Target = ListEntry<T>;
+
+ fn as_raw(handle: &Self::Handle) -> NonNull<ListEntry<T>> {
+ let ptr: *const ListEntry<T> = Arc::as_ptr(handle);
+ // Safety: We can't get a null pointer from `Arc::as_ptr`.
+ unsafe { NonNull::new_unchecked(ptr as *mut ListEntry<T>) }
+ }
+
+ unsafe fn from_raw(ptr: NonNull<ListEntry<T>>) -> Arc<ListEntry<T>> {
+ Arc::from_raw(ptr.as_ptr())
+ }
+
+ unsafe fn pointers(
+ target: NonNull<ListEntry<T>>,
+ ) -> NonNull<linked_list::Pointers<ListEntry<T>>> {
+ ListEntry::addr_of_pointers(target)
+ }
+}
diff --git a/src/util/linked_list.rs b/src/util/linked_list.rs
index 894d216..b46bd6d 100644
--- a/src/util/linked_list.rs
+++ b/src/util/linked_list.rs
@@ -57,6 +57,13 @@
unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Self::Handle;
/// Return the pointers for a node
+ ///
+ /// # Safety
+ ///
+ /// The resulting pointer should have the same tag in the stacked-borrows
+ /// stack as the argument. In particular, the method may not create an
+ /// intermediate reference in the process of creating the resulting raw
+ /// pointer.
unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>>;
}
@@ -119,7 +126,7 @@
pub(crate) fn push_front(&mut self, val: L::Handle) {
// The value should not be dropped, it is being inserted into the list
let val = ManuallyDrop::new(val);
- let ptr = L::as_raw(&*val);
+ let ptr = L::as_raw(&val);
assert_ne!(self.head, Some(ptr));
unsafe {
L::pointers(ptr).as_mut().set_next(self.head);
@@ -219,6 +226,7 @@
#[cfg(any(
feature = "fs",
+ feature = "rt",
all(unix, feature = "process"),
feature = "signal",
feature = "sync",
@@ -296,7 +304,7 @@
}
}
- fn get_prev(&self) -> Option<NonNull<T>> {
+ pub(crate) fn get_prev(&self) -> Option<NonNull<T>> {
// SAFETY: prev is the first field in PointersInner, which is #[repr(C)].
unsafe {
let inner = self.inner.get();
@@ -304,7 +312,7 @@
ptr::read(prev)
}
}
- fn get_next(&self) -> Option<NonNull<T>> {
+ pub(crate) fn get_next(&self) -> Option<NonNull<T>> {
// SAFETY: next is the second field in PointersInner, which is #[repr(C)].
unsafe {
let inner = self.inner.get();
@@ -352,6 +360,7 @@
use std::pin::Pin;
#[derive(Debug)]
+ #[repr(C)]
struct Entry {
pointers: Pointers<Entry>,
val: i32,
@@ -369,8 +378,8 @@
Pin::new_unchecked(&*ptr.as_ptr())
}
- unsafe fn pointers(mut target: NonNull<Entry>) -> NonNull<Pointers<Entry>> {
- NonNull::from(&mut target.as_mut().pointers)
+ unsafe fn pointers(target: NonNull<Entry>) -> NonNull<Pointers<Entry>> {
+ target.cast()
}
}
@@ -614,6 +623,7 @@
}
}
+ #[cfg(not(tokio_wasm))]
proptest::proptest! {
#[test]
fn fuzz_linked_list(ops: Vec<usize>) {
@@ -621,6 +631,7 @@
}
}
+ #[cfg(not(tokio_wasm))]
fn run_fuzz(ops: Vec<usize>) {
use std::collections::VecDeque;
diff --git a/src/util/mod.rs b/src/util/mod.rs
index df30f2b..9f6119a 100644
--- a/src/util/mod.rs
+++ b/src/util/mod.rs
@@ -3,6 +3,17 @@
pub(crate) mod slab;
}
+#[cfg(feature = "rt")]
+pub(crate) mod atomic_cell;
+
+#[cfg(any(
+ feature = "rt",
+ feature = "signal",
+ feature = "process",
+ tokio_no_const_mutex_new,
+))]
+pub(crate) mod once_cell;
+
#[cfg(any(
// io driver uses `WakeList` directly
feature = "net",
@@ -37,10 +48,15 @@
))]
pub(crate) mod linked_list;
-#[cfg(any(feature = "rt-multi-thread", feature = "macros"))]
-mod rand;
+#[cfg(any(feature = "rt", feature = "macros"))]
+pub(crate) mod rand;
cfg_rt! {
+ mod idle_notified_set;
+ pub(crate) use idle_notified_set::IdleNotifiedSet;
+
+ pub(crate) use self::rand::RngSeedGenerator;
+
mod wake;
pub(crate) use wake::WakerRef;
pub(crate) use wake::{waker_ref, Wake};
@@ -48,28 +64,15 @@
mod sync_wrapper;
pub(crate) use sync_wrapper::SyncWrapper;
- mod vec_deque_cell;
- pub(crate) use vec_deque_cell::VecDequeCell;
+ mod rc_cell;
+ pub(crate) use rc_cell::RcCell;
}
cfg_rt_multi_thread! {
- pub(crate) use self::rand::FastRand;
-
mod try_lock;
pub(crate) use try_lock::TryLock;
}
pub(crate) mod trace;
-#[cfg(any(feature = "macros"))]
-#[cfg_attr(not(feature = "macros"), allow(unreachable_pub))]
-pub use self::rand::thread_rng_n;
-
-#[cfg(any(
- feature = "rt",
- feature = "time",
- feature = "net",
- feature = "process",
- all(unix, feature = "signal")
-))]
pub(crate) mod error;
diff --git a/src/util/once_cell.rs b/src/util/once_cell.rs
new file mode 100644
index 0000000..1925f0a
--- /dev/null
+++ b/src/util/once_cell.rs
@@ -0,0 +1,70 @@
+#![allow(dead_code)]
+use std::cell::UnsafeCell;
+use std::mem::MaybeUninit;
+use std::sync::Once;
+
+pub(crate) struct OnceCell<T> {
+ once: Once,
+ value: UnsafeCell<MaybeUninit<T>>,
+}
+
+unsafe impl<T: Send + Sync> Send for OnceCell<T> {}
+unsafe impl<T: Send + Sync> Sync for OnceCell<T> {}
+
+impl<T> OnceCell<T> {
+ pub(crate) const fn new() -> Self {
+ Self {
+ once: Once::new(),
+ value: UnsafeCell::new(MaybeUninit::uninit()),
+ }
+ }
+
+ /// Get the value inside this cell, intiailizing it using the provided
+ /// function if necessary.
+ ///
+ /// If the `init` closure panics, then the `OnceCell` is poisoned and all
+ /// future calls to `get` will panic.
+ #[inline]
+ pub(crate) fn get(&self, init: impl FnOnce() -> T) -> &T {
+ if !self.once.is_completed() {
+ self.do_init(init);
+ }
+
+ // Safety: The `std::sync::Once` guarantees that we can only reach this
+ // line if a `call_once` closure has been run exactly once and without
+ // panicking. Thus, the value is not uninitialized.
+ //
+ // There is also no race because the only `&self` method that modifies
+ // `value` is `do_init`, but if the `call_once` closure is still
+ // running, then no thread has gotten past the `call_once`.
+ unsafe { &*(self.value.get() as *const T) }
+ }
+
+ #[cold]
+ fn do_init(&self, init: impl FnOnce() -> T) {
+ let value_ptr = self.value.get() as *mut T;
+
+ self.once.call_once(|| {
+ let set_to = init();
+
+ // Safety: The `std::sync::Once` guarantees that this initialization
+ // will run at most once, and that no thread can get past the
+ // `call_once` until it has run exactly once. Thus, we have
+ // exclusive access to `value`.
+ unsafe {
+ std::ptr::write(value_ptr, set_to);
+ }
+ });
+ }
+}
+
+impl<T> Drop for OnceCell<T> {
+ fn drop(&mut self) {
+ if self.once.is_completed() {
+ let value_ptr = self.value.get() as *mut T;
+ unsafe {
+ std::ptr::drop_in_place(value_ptr);
+ }
+ }
+ }
+}
diff --git a/src/util/rand.rs b/src/util/rand.rs
index 6b19c8b..749da6b 100644
--- a/src/util/rand.rs
+++ b/src/util/rand.rs
@@ -1,5 +1,103 @@
use std::cell::Cell;
+cfg_rt! {
+ use std::sync::Mutex;
+
+ /// A deterministic generator for seeds (and other generators).
+ ///
+ /// Given the same initial seed, the generator will output the same sequence of seeds.
+ ///
+ /// Since the seed generator will be kept in a runtime handle, we need to wrap `FastRand`
+ /// in a Mutex to make it thread safe. Different to the `FastRand` that we keep in a
+ /// thread local store, the expectation is that seed generation will not need to happen
+ /// very frequently, so the cost of the mutex should be minimal.
+ #[derive(Debug)]
+ pub(crate) struct RngSeedGenerator {
+ /// Internal state for the seed generator. We keep it in a Mutex so that we can safely
+ /// use it across multiple threads.
+ state: Mutex<FastRand>,
+ }
+
+ impl RngSeedGenerator {
+ /// Returns a new generator from the provided seed.
+ pub(crate) fn new(seed: RngSeed) -> Self {
+ Self {
+ state: Mutex::new(FastRand::new(seed)),
+ }
+ }
+
+ /// Returns the next seed in the sequence.
+ pub(crate) fn next_seed(&self) -> RngSeed {
+ let rng = self
+ .state
+ .lock()
+ .expect("RNG seed generator is internally corrupt");
+
+ let s = rng.fastrand();
+ let r = rng.fastrand();
+
+ RngSeed::from_pair(s, r)
+ }
+
+ /// Directly creates a generator using the next seed.
+ pub(crate) fn next_generator(&self) -> Self {
+ RngSeedGenerator::new(self.next_seed())
+ }
+ }
+}
+
+/// A seed for random number generation.
+///
+/// In order to make certain functions within a runtime deterministic, a seed
+/// can be specified at the time of creation.
+#[allow(unreachable_pub)]
+#[derive(Clone, Debug)]
+pub struct RngSeed {
+ s: u32,
+ r: u32,
+}
+
+impl RngSeed {
+ /// Creates a random seed using loom internally.
+ pub(crate) fn new() -> Self {
+ Self::from_u64(crate::loom::rand::seed())
+ }
+
+ cfg_unstable! {
+ /// Generates a seed from the provided byte slice.
+ ///
+ /// # Example
+ ///
+ /// ```
+ /// # use tokio::runtime::RngSeed;
+ /// let seed = RngSeed::from_bytes(b"make me a seed");
+ /// ```
+ #[cfg(feature = "rt")]
+ pub fn from_bytes(bytes: &[u8]) -> Self {
+ use std::{collections::hash_map::DefaultHasher, hash::Hasher};
+
+ let mut hasher = DefaultHasher::default();
+ hasher.write(bytes);
+ Self::from_u64(hasher.finish())
+ }
+ }
+
+ fn from_u64(seed: u64) -> Self {
+ let one = (seed >> 32) as u32;
+ let mut two = seed as u32;
+
+ if two == 0 {
+ // This value cannot be zero
+ two = 1;
+ }
+
+ Self::from_pair(one, two)
+ }
+
+ fn from_pair(s: u32, r: u32) -> Self {
+ Self { s, r }
+ }
+}
/// Fast random number generate.
///
/// Implement xorshift64+: 2 32-bit xorshift sequences added together.
@@ -15,21 +113,29 @@
impl FastRand {
/// Initializes a new, thread-local, fast random number generator.
- pub(crate) fn new(seed: u64) -> FastRand {
- let one = (seed >> 32) as u32;
- let mut two = seed as u32;
-
- if two == 0 {
- // This value cannot be zero
- two = 1;
- }
-
+ pub(crate) fn new(seed: RngSeed) -> FastRand {
FastRand {
- one: Cell::new(one),
- two: Cell::new(two),
+ one: Cell::new(seed.s),
+ two: Cell::new(seed.r),
}
}
+ /// Replaces the state of the random number generator with the provided seed, returning
+ /// the seed that represents the previous state of the random number generator.
+ ///
+ /// The random number generator will become equivalent to one created with
+ /// the same seed.
+ #[cfg(feature = "rt")]
+ pub(crate) fn replace_seed(&self, seed: RngSeed) -> RngSeed {
+ let old_seed = RngSeed::from_pair(self.one.get(), self.two.get());
+
+ self.one.replace(seed.s);
+ self.two.replace(seed.r);
+
+ old_seed
+ }
+
+ #[cfg(any(feature = "macros", feature = "rt-multi-thread"))]
pub(crate) fn fastrand_n(&self, n: u32) -> u32 {
// This is similar to fastrand() % n, but faster.
// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
@@ -50,15 +156,3 @@
s0.wrapping_add(s1)
}
}
-
-// Used by the select macro and `StreamMap`
-#[cfg(any(feature = "macros"))]
-#[doc(hidden)]
-#[cfg_attr(not(feature = "macros"), allow(unreachable_pub))]
-pub fn thread_rng_n(n: u32) -> u32 {
- thread_local! {
- static THREAD_RNG: FastRand = FastRand::new(crate::loom::rand::seed());
- }
-
- THREAD_RNG.with(|rng| rng.fastrand_n(n))
-}
diff --git a/src/util/rc_cell.rs b/src/util/rc_cell.rs
new file mode 100644
index 0000000..4472492
--- /dev/null
+++ b/src/util/rc_cell.rs
@@ -0,0 +1,57 @@
+use crate::loom::cell::UnsafeCell;
+
+use std::rc::Rc;
+
+/// This is exactly like `Cell<Option<Rc<T>>>`, except that it provides a `get`
+/// method even though `Rc` is not `Copy`.
+pub(crate) struct RcCell<T> {
+ inner: UnsafeCell<Option<Rc<T>>>,
+}
+
+impl<T> RcCell<T> {
+ #[cfg(not(all(loom, test)))]
+ pub(crate) const fn new() -> Self {
+ Self {
+ inner: UnsafeCell::new(None),
+ }
+ }
+
+ // The UnsafeCell in loom does not have a const `new` fn.
+ #[cfg(all(loom, test))]
+ pub(crate) fn new() -> Self {
+ Self {
+ inner: UnsafeCell::new(None),
+ }
+ }
+
+ /// Safety: This method may not be called recursively.
+ #[inline]
+ unsafe fn with_inner<F, R>(&self, f: F) -> R
+ where
+ F: FnOnce(&mut Option<Rc<T>>) -> R,
+ {
+ // safety: This type is not Sync, so concurrent calls of this method
+ // cannot happen. Furthermore, the caller guarantees that the method is
+ // not called recursively. Finally, this is the only place that can
+ // create mutable references to the inner Rc. This ensures that any
+ // mutable references created here are exclusive.
+ self.inner.with_mut(|ptr| f(&mut *ptr))
+ }
+
+ pub(crate) fn get(&self) -> Option<Rc<T>> {
+ // safety: The `Rc::clone` method will not call any unknown user-code,
+ // so it will not result in a recursive call to `with_inner`.
+ unsafe { self.with_inner(|rc| rc.clone()) }
+ }
+
+ pub(crate) fn replace(&self, val: Option<Rc<T>>) -> Option<Rc<T>> {
+ // safety: No destructors or other unknown user-code will run inside the
+ // `with_inner` call, so no recursive call to `with_inner` can happen.
+ unsafe { self.with_inner(|rc| std::mem::replace(rc, val)) }
+ }
+
+ pub(crate) fn set(&self, val: Option<Rc<T>>) {
+ let old = self.replace(val);
+ drop(old);
+ }
+}
diff --git a/src/util/slab.rs b/src/util/slab.rs
index 97355d5..0e16e40 100644
--- a/src/util/slab.rs
+++ b/src/util/slab.rs
@@ -157,6 +157,12 @@
/// Next entry in the free list.
next: u32,
+
+ /// Makes miri happy by making mutable references not take exclusive access.
+ ///
+ /// Could probably also be fixed by replacing `slots` with a raw-pointer
+ /// based equivalent.
+ _pin: std::marker::PhantomPinned,
}
/// Value paired with a reference to the page.
@@ -409,7 +415,7 @@
slot.value.with(|ptr| unsafe { (*ptr).value.reset() });
// Return a reference to the slot
- Some((me.addr(idx), slot.gen_ref(me)))
+ Some((me.addr(idx), locked.gen_ref(idx, me)))
} else if me.len == locked.slots.len() {
// The page is full
None
@@ -428,9 +434,10 @@
locked.slots.push(Slot {
value: UnsafeCell::new(Value {
value: Default::default(),
- page: &**me as *const _,
+ page: Arc::as_ptr(me),
}),
next: 0,
+ _pin: std::marker::PhantomPinned,
});
// Increment the head to indicate the free stack is empty
@@ -443,7 +450,7 @@
debug_assert_eq!(locked.slots.len(), locked.head);
- Some((me.addr(idx), locked.slots[idx].gen_ref(me)))
+ Some((me.addr(idx), locked.gen_ref(idx, me)))
}
}
}
@@ -544,10 +551,9 @@
fn index_for(&self, slot: *const Value<T>) -> usize {
use std::mem;
- let base = &self.slots[0] as *const _ as usize;
+ assert_ne!(self.slots.capacity(), 0, "page is unallocated");
- assert!(base != 0, "page is unallocated");
-
+ let base = self.slots.as_ptr() as usize;
let slot = slot as usize;
let width = mem::size_of::<Slot<T>>();
@@ -558,18 +564,15 @@
idx
}
-}
-impl<T: Entry> Slot<T> {
- /// Generates a `Ref` for the slot. This involves bumping the page's ref count.
- fn gen_ref(&self, page: &Arc<Page<T>>) -> Ref<T> {
- // The ref holds a ref on the page. The `Arc` is forgotten here and is
- // resurrected in `release` when the `Ref` is dropped. By avoiding to
- // hold on to an explicit `Arc` value, the struct size of `Ref` is
- // reduced.
+ /// Generates a `Ref` for the slot at the given index. This involves bumping the page's ref count.
+ fn gen_ref(&self, idx: usize, page: &Arc<Page<T>>) -> Ref<T> {
+ assert!(idx < self.slots.len());
mem::forget(page.clone());
- let slot = self as *const Slot<T>;
- let value = slot as *const Value<T>;
+
+ let vec_ptr = self.slots.as_ptr();
+ let slot: *const Slot<T> = unsafe { vec_ptr.add(idx) };
+ let value: *const Value<T> = slot as *const Value<T>;
Ref { value }
}
@@ -691,11 +694,13 @@
#[test]
fn insert_many() {
+ const MANY: usize = normal_or_miri(10_000, 50);
+
let mut slab = Slab::<Foo>::new();
let alloc = slab.allocator();
let mut entries = vec![];
- for i in 0..10_000 {
+ for i in 0..MANY {
let (addr, val) = alloc.allocate().unwrap();
val.id.store(i, SeqCst);
entries.push((addr, val));
@@ -708,15 +713,15 @@
entries.clear();
- for i in 0..10_000 {
+ for i in 0..MANY {
let (addr, val) = alloc.allocate().unwrap();
- val.id.store(10_000 - i, SeqCst);
+ val.id.store(MANY - i, SeqCst);
entries.push((addr, val));
}
for (i, (addr, v)) in entries.iter().enumerate() {
- assert_eq!(10_000 - i, v.id.load(SeqCst));
- assert_eq!(10_000 - i, slab.get(*addr).unwrap().id.load(SeqCst));
+ assert_eq!(MANY - i, v.id.load(SeqCst));
+ assert_eq!(MANY - i, slab.get(*addr).unwrap().id.load(SeqCst));
}
}
@@ -726,7 +731,7 @@
let alloc = slab.allocator();
let mut entries = vec![];
- for i in 0..10_000 {
+ for i in 0..normal_or_miri(10_000, 100) {
let (addr, val) = alloc.allocate().unwrap();
val.id.store(i, SeqCst);
entries.push((addr, val));
@@ -734,7 +739,7 @@
for _ in 0..10 {
// Drop 1000 in reverse
- for _ in 0..1_000 {
+ for _ in 0..normal_or_miri(1_000, 10) {
entries.pop();
}
@@ -753,7 +758,7 @@
let mut entries1 = vec![];
let mut entries2 = vec![];
- for i in 0..10_000 {
+ for i in 0..normal_or_miri(10_000, 100) {
let (addr, val) = alloc.allocate().unwrap();
val.id.store(i, SeqCst);
@@ -771,6 +776,14 @@
}
}
+ const fn normal_or_miri(normal: usize, miri: usize) -> usize {
+ if cfg!(miri) {
+ miri
+ } else {
+ normal
+ }
+ }
+
#[test]
fn compact_all() {
let mut slab = Slab::<Foo>::new();
@@ -780,7 +793,7 @@
for _ in 0..2 {
entries.clear();
- for i in 0..10_000 {
+ for i in 0..normal_or_miri(10_000, 100) {
let (addr, val) = alloc.allocate().unwrap();
val.id.store(i, SeqCst);
@@ -808,7 +821,7 @@
let alloc = slab.allocator();
let mut entries = vec![];
- for _ in 0..5 {
+ for _ in 0..normal_or_miri(5, 2) {
entries.clear();
// Allocate a few pages + 1
diff --git a/src/util/trace.rs b/src/util/trace.rs
index e3c26f9..76e8a6c 100644
--- a/src/util/trace.rs
+++ b/src/util/trace.rs
@@ -1,40 +1,90 @@
cfg_trace! {
cfg_rt! {
+ use core::{
+ pin::Pin,
+ task::{Context, Poll},
+ };
+ use pin_project_lite::pin_project;
+ use std::future::Future;
pub(crate) use tracing::instrument::Instrumented;
#[inline]
- #[cfg_attr(tokio_track_caller, track_caller)]
- pub(crate) fn task<F>(task: F, kind: &'static str, name: Option<&str>) -> Instrumented<F> {
+ #[track_caller]
+ pub(crate) fn task<F>(task: F, kind: &'static str, name: Option<&str>, id: u64) -> Instrumented<F> {
use tracing::instrument::Instrument;
- #[cfg(tokio_track_caller)]
let location = std::panic::Location::caller();
- #[cfg(tokio_track_caller)]
let span = tracing::trace_span!(
target: "tokio::task",
"runtime.spawn",
%kind,
task.name = %name.unwrap_or_default(),
+ task.id = id,
loc.file = location.file(),
loc.line = location.line(),
loc.col = location.column(),
);
- #[cfg(not(tokio_track_caller))]
- let span = tracing::trace_span!(
- target: "tokio::task",
- "runtime.spawn",
- %kind,
- task.name = %name.unwrap_or_default(),
- );
task.instrument(span)
}
+
+ pub(crate) fn async_op<P,F>(inner: P, resource_span: tracing::Span, source: &str, poll_op_name: &'static str, inherits_child_attrs: bool) -> InstrumentedAsyncOp<F>
+ where P: FnOnce() -> F {
+ resource_span.in_scope(|| {
+ let async_op_span = tracing::trace_span!("runtime.resource.async_op", source = source, inherits_child_attrs = inherits_child_attrs);
+ let enter = async_op_span.enter();
+ let async_op_poll_span = tracing::trace_span!("runtime.resource.async_op.poll");
+ let inner = inner();
+ drop(enter);
+ let tracing_ctx = AsyncOpTracingCtx {
+ async_op_span,
+ async_op_poll_span,
+ resource_span: resource_span.clone(),
+ };
+ InstrumentedAsyncOp {
+ inner,
+ tracing_ctx,
+ poll_op_name,
+ }
+ })
+ }
+
+ #[derive(Debug, Clone)]
+ pub(crate) struct AsyncOpTracingCtx {
+ pub(crate) async_op_span: tracing::Span,
+ pub(crate) async_op_poll_span: tracing::Span,
+ pub(crate) resource_span: tracing::Span,
+ }
+
+
+ pin_project! {
+ #[derive(Debug, Clone)]
+ pub(crate) struct InstrumentedAsyncOp<F> {
+ #[pin]
+ pub(crate) inner: F,
+ pub(crate) tracing_ctx: AsyncOpTracingCtx,
+ pub(crate) poll_op_name: &'static str
+ }
+ }
+
+ impl<F: Future> Future for InstrumentedAsyncOp<F> {
+ type Output = F::Output;
+
+ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+ let this = self.project();
+ let poll_op_name = &*this.poll_op_name;
+ let _res_enter = this.tracing_ctx.resource_span.enter();
+ let _async_op_enter = this.tracing_ctx.async_op_span.enter();
+ let _async_op_poll_enter = this.tracing_ctx.async_op_poll_span.enter();
+ trace_poll_op!(poll_op_name, this.inner.poll(cx))
+ }
+ }
}
}
cfg_time! {
- #[cfg_attr(tokio_track_caller, track_caller)]
+ #[track_caller]
pub(crate) fn caller_location() -> Option<&'static std::panic::Location<'static>> {
- #[cfg(all(tokio_track_caller, tokio_unstable, feature = "tracing"))]
+ #[cfg(all(tokio_unstable, feature = "tracing"))]
return Some(std::panic::Location::caller());
- #[cfg(not(all(tokio_track_caller, tokio_unstable, feature = "tracing")))]
+ #[cfg(not(all(tokio_unstable, feature = "tracing")))]
None
}
}
@@ -42,7 +92,7 @@
cfg_not_trace! {
cfg_rt! {
#[inline]
- pub(crate) fn task<F>(task: F, _: &'static str, _name: Option<&str>) -> F {
+ pub(crate) fn task<F>(task: F, _: &'static str, _name: Option<&str>, _: u64) -> F {
// nop
task
}
diff --git a/src/util/vec_deque_cell.rs b/src/util/vec_deque_cell.rs
deleted file mode 100644
index b4e124c..0000000
--- a/src/util/vec_deque_cell.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-use crate::loom::cell::UnsafeCell;
-
-use std::collections::VecDeque;
-use std::marker::PhantomData;
-
-/// This type is like VecDeque, except that it is not Sync and can be modified
-/// through immutable references.
-pub(crate) struct VecDequeCell<T> {
- inner: UnsafeCell<VecDeque<T>>,
- _not_sync: PhantomData<*const ()>,
-}
-
-// This is Send for the same reasons that RefCell<VecDeque<T>> is Send.
-unsafe impl<T: Send> Send for VecDequeCell<T> {}
-
-impl<T> VecDequeCell<T> {
- pub(crate) fn with_capacity(cap: usize) -> Self {
- Self {
- inner: UnsafeCell::new(VecDeque::with_capacity(cap)),
- _not_sync: PhantomData,
- }
- }
-
- /// Safety: This method may not be called recursively.
- #[inline]
- unsafe fn with_inner<F, R>(&self, f: F) -> R
- where
- F: FnOnce(&mut VecDeque<T>) -> R,
- {
- // safety: This type is not Sync, so concurrent calls of this method
- // cannot happen. Furthermore, the caller guarantees that the method is
- // not called recursively. Finally, this is the only place that can
- // create mutable references to the inner VecDeque. This ensures that
- // any mutable references created here are exclusive.
- self.inner.with_mut(|ptr| f(&mut *ptr))
- }
-
- pub(crate) fn pop_front(&self) -> Option<T> {
- unsafe { self.with_inner(VecDeque::pop_front) }
- }
-
- pub(crate) fn push_back(&self, item: T) {
- unsafe {
- self.with_inner(|inner| inner.push_back(item));
- }
- }
-
- /// Replaces the inner VecDeque with an empty VecDeque and return the current
- /// contents.
- pub(crate) fn take(&self) -> VecDeque<T> {
- unsafe { self.with_inner(|inner| std::mem::take(inner)) }
- }
-}
diff --git a/src/util/wake.rs b/src/util/wake.rs
index 8f89668..5526cbc 100644
--- a/src/util/wake.rs
+++ b/src/util/wake.rs
@@ -1,13 +1,14 @@
+use crate::loom::sync::Arc;
+
use std::marker::PhantomData;
use std::mem::ManuallyDrop;
use std::ops::Deref;
-use std::sync::Arc;
use std::task::{RawWaker, RawWakerVTable, Waker};
/// Simplified waking interface based on Arcs.
-pub(crate) trait Wake: Send + Sync {
+pub(crate) trait Wake: Send + Sync + Sized + 'static {
/// Wake by value.
- fn wake(self: Arc<Self>);
+ fn wake(arc_self: Arc<Self>);
/// Wake by reference.
fn wake_by_ref(arc_self: &Arc<Self>);
@@ -30,7 +31,7 @@
/// Creates a reference to a `Waker` from a reference to `Arc<impl Wake>`.
pub(crate) fn waker_ref<W: Wake>(wake: &Arc<W>) -> WakerRef<'_> {
- let ptr = &**wake as *const _ as *const ();
+ let ptr = Arc::as_ptr(wake) as *const ();
let waker = unsafe { Waker::from_raw(RawWaker::new(ptr, waker_vtable::<W>())) };
diff --git a/tests/_require_full.rs b/tests/_require_full.rs
index 98455be..4b9698a 100644
--- a/tests/_require_full.rs
+++ b/tests/_require_full.rs
@@ -1,2 +1,8 @@
-#![cfg(not(feature = "full"))]
+#[cfg(not(any(feature = "full", tokio_wasm)))]
compile_error!("run main Tokio tests with `--features full`");
+
+// CI sets `--cfg tokio_no_parking_lot` when trying to run tests with
+// `parking_lot` disabled. This check prevents "silent failure" if `parking_lot`
+// accidentally gets enabled.
+#[cfg(all(tokio_no_parking_lot, feature = "parking_lot"))]
+compile_error!("parking_lot feature enabled when it should not be");
diff --git a/tests/async_send_sync.rs b/tests/async_send_sync.rs
index aa14970..e46d5c8 100644
--- a/tests/async_send_sync.rs
+++ b/tests/async_send_sync.rs
@@ -127,70 +127,111 @@
};
}
-assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin);
-assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin);
-assert_value!(tokio::fs::File: Send & Sync & Unpin);
-assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin);
-assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin);
+macro_rules! cfg_not_wasi {
+ ($($item:item)*) => {
+ $(
+ #[cfg(not(tokio_wasi))]
+ $item
+ )*
+ }
+}
-async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::write(&str, Vec<u8>): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin);
+// Manually re-implementation of `async_assert_fn` for `poll_fn`. The macro
+// doesn't work for this particular case because constructing the closure
+// is too complicated.
+const _: fn() = || {
+ let pinned = std::marker::PhantomPinned;
+ let f = tokio::macros::support::poll_fn(move |_| {
+ // Use `pinned` to take ownership of it.
+ let _ = &pinned;
+ std::task::Poll::Pending::<()>
+ });
+ require_send(&f);
+ require_sync(&f);
+ AmbiguousIfUnpin::some_item(&f);
+};
+
+cfg_not_wasi! {
+ mod fs {
+ use super::*;
+ assert_value!(tokio::fs::DirBuilder: Send & Sync & Unpin);
+ assert_value!(tokio::fs::DirEntry: Send & Sync & Unpin);
+ assert_value!(tokio::fs::File: Send & Sync & Unpin);
+ assert_value!(tokio::fs::OpenOptions: Send & Sync & Unpin);
+ assert_value!(tokio::fs::ReadDir: Send & Sync & Unpin);
+
+ async_assert_fn!(tokio::fs::canonicalize(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::copy(&str, &str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::create_dir(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::create_dir_all(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::hard_link(&str, &str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::metadata(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::read(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::read_dir(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::read_link(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::read_to_string(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::remove_dir(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::remove_dir_all(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::remove_file(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::rename(&str, &str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::set_permissions(&str, std::fs::Permissions): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::symlink_metadata(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::write(&str, Vec<u8>): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::ReadDir::next_entry(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::OpenOptions::open(_, &str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::DirBuilder::create(_, &str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::DirEntry::metadata(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::DirEntry::file_type(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::open(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::create(&str): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::sync_all(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::sync_data(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::set_len(_, u64): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::metadata(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::try_clone(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::fs::File::into_std(_): Send & Sync & !Unpin);
+ async_assert_fn!(
+ tokio::fs::File::set_permissions(_, std::fs::Permissions): Send & Sync & !Unpin
+ );
+ }
+}
+
+cfg_not_wasi! {
+ assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin);
+ async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin);
+}
assert_value!(tokio::net::TcpListener: Send & Sync & Unpin);
-assert_value!(tokio::net::TcpSocket: Send & Sync & Unpin);
assert_value!(tokio::net::TcpStream: Send & Sync & Unpin);
-assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin);
assert_value!(tokio::net::tcp::OwnedReadHalf: Send & Sync & Unpin);
assert_value!(tokio::net::tcp::OwnedWriteHalf: Send & Sync & Unpin);
assert_value!(tokio::net::tcp::ReadHalf<'_>: Send & Sync & Unpin);
assert_value!(tokio::net::tcp::ReuniteError: Send & Sync & Unpin);
assert_value!(tokio::net::tcp::WriteHalf<'_>: Send & Sync & Unpin);
async_assert_fn!(tokio::net::TcpListener::accept(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::TcpListener::bind(SocketAddr): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::TcpStream::connect(SocketAddr): Send & Sync & !Unpin);
async_assert_fn!(tokio::net::TcpStream::peek(_, &mut [u8]): Send & Sync & !Unpin);
async_assert_fn!(tokio::net::TcpStream::readable(_): Send & Sync & !Unpin);
async_assert_fn!(tokio::net::TcpStream::ready(_, tokio::io::Interest): Send & Sync & !Unpin);
async_assert_fn!(tokio::net::TcpStream::writable(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin);
-async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin);
+
+// Wasi does not support UDP
+cfg_not_wasi! {
+ mod udp_socket {
+ use super::*;
+ assert_value!(tokio::net::UdpSocket: Send & Sync & Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::bind(SocketAddr): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::connect(_, SocketAddr): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::peek_from(_, &mut [u8]): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::readable(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::ready(_, tokio::io::Interest): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::recv(_, &mut [u8]): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::recv_from(_, &mut [u8]): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::send(_, &[u8]): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::send_to(_, &[u8], SocketAddr): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::net::UdpSocket::writable(_): Send & Sync & !Unpin);
+ }
+}
async_assert_fn!(tokio::net::lookup_host(SocketAddr): Send & Sync & !Unpin);
async_assert_fn!(tokio::net::tcp::ReadHalf::peek(_, &mut [u8]): Send & Sync & !Unpin);
@@ -242,16 +283,22 @@
async_assert_fn!(NamedPipeServer::writable(_): Send & Sync & !Unpin);
}
-assert_value!(tokio::process::Child: Send & Sync & Unpin);
-assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin);
-assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin);
-assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin);
-assert_value!(tokio::process::Command: Send & Sync & Unpin);
-async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin);
-async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin);
+cfg_not_wasi! {
+ mod test_process {
+ use super::*;
+ assert_value!(tokio::process::Child: Send & Sync & Unpin);
+ assert_value!(tokio::process::ChildStderr: Send & Sync & Unpin);
+ assert_value!(tokio::process::ChildStdin: Send & Sync & Unpin);
+ assert_value!(tokio::process::ChildStdout: Send & Sync & Unpin);
+ assert_value!(tokio::process::Command: Send & Sync & Unpin);
+ async_assert_fn!(tokio::process::Child::kill(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::process::Child::wait(_): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::process::Child::wait_with_output(_): Send & Sync & !Unpin);
+ }
-async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin);
+ async_assert_fn!(tokio::signal::ctrl_c(): Send & Sync & !Unpin);
+}
+
#[cfg(unix)]
mod unix_signal {
use super::*;
@@ -362,6 +409,14 @@
assert_value!(tokio::sync::watch::Sender<NN>: !Send & !Sync & Unpin);
assert_value!(tokio::sync::watch::Sender<YN>: !Send & !Sync & Unpin);
assert_value!(tokio::sync::watch::Sender<YY>: Send & Sync & Unpin);
+assert_value!(tokio::task::JoinError: Send & Sync & Unpin);
+assert_value!(tokio::task::JoinHandle<NN>: !Send & !Sync & Unpin);
+assert_value!(tokio::task::JoinHandle<YN>: Send & Sync & Unpin);
+assert_value!(tokio::task::JoinHandle<YY>: Send & Sync & Unpin);
+assert_value!(tokio::task::JoinSet<NN>: !Send & !Sync & Unpin);
+assert_value!(tokio::task::JoinSet<YN>: Send & Sync & Unpin);
+assert_value!(tokio::task::JoinSet<YY>: Send & Sync & Unpin);
+assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin);
async_assert_fn!(tokio::sync::Barrier::wait(_): Send & Sync & !Unpin);
async_assert_fn!(tokio::sync::Mutex<NN>::lock(_): !Send & !Sync & !Unpin);
async_assert_fn!(tokio::sync::Mutex<NN>::lock_owned(_): !Send & !Sync & !Unpin);
@@ -434,25 +489,25 @@
async_assert_fn!(tokio::sync::watch::Sender<NN>::closed(_): !Send & !Sync & !Unpin);
async_assert_fn!(tokio::sync::watch::Sender<YN>::closed(_): !Send & !Sync & !Unpin);
async_assert_fn!(tokio::sync::watch::Sender<YY>::closed(_): Send & Sync & !Unpin);
-
-async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSync<()>): Send & Sync & !Unpin);
-async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSend<()>): Send & !Sync & !Unpin);
-async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFuture<()>): !Send & !Sync & !Unpin);
-async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSync<()>): Send & !Sync & !Unpin);
-async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSend<()>): Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::JoinSet<Cell<u32>>::join_next(_): Send & Sync & !Unpin);
+async_assert_fn!(tokio::task::JoinSet<Cell<u32>>::shutdown(_): Send & Sync & !Unpin);
+async_assert_fn!(tokio::task::JoinSet<Rc<u32>>::join_next(_): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::JoinSet<Rc<u32>>::shutdown(_): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::JoinSet<u32>::join_next(_): Send & Sync & !Unpin);
+async_assert_fn!(tokio::task::JoinSet<u32>::shutdown(_): Send & Sync & !Unpin);
async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFuture<()>): !Send & !Sync & !Unpin);
-async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSync<()>): !Send & !Sync & !Unpin);
-async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSend<()>): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSend<()>): Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<Cell<u32>>::scope(_, Cell<u32>, BoxFutureSync<()>): Send & !Sync & !Unpin);
async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFuture<()>): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSend<()>): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<Rc<u32>>::scope(_, Rc<u32>, BoxFutureSync<()>): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFuture<()>): !Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSend<()>): Send & !Sync & !Unpin);
+async_assert_fn!(tokio::task::LocalKey<u32>::scope(_, u32, BoxFutureSync<()>): Send & Sync & !Unpin);
async_assert_fn!(tokio::task::LocalSet::run_until(_, BoxFutureSync<()>): !Send & !Sync & !Unpin);
async_assert_fn!(tokio::task::unconstrained(BoxFuture<()>): !Send & !Sync & Unpin);
async_assert_fn!(tokio::task::unconstrained(BoxFutureSend<()>): Send & !Sync & Unpin);
async_assert_fn!(tokio::task::unconstrained(BoxFutureSync<()>): Send & Sync & Unpin);
-assert_value!(tokio::task::LocalSet: !Send & !Sync & Unpin);
-assert_value!(tokio::task::JoinHandle<YY>: Send & Sync & Unpin);
-assert_value!(tokio::task::JoinHandle<YN>: Send & Sync & Unpin);
-assert_value!(tokio::task::JoinHandle<NN>: !Send & !Sync & Unpin);
-assert_value!(tokio::task::JoinError: Send & Sync & Unpin);
assert_value!(tokio::runtime::Builder: Send & Sync & Unpin);
assert_value!(tokio::runtime::EnterGuard<'_>: Send & Sync & Unpin);
diff --git a/tests/buffered.rs b/tests/buffered.rs
index 98b6d5f..4251c3f 100644
--- a/tests/buffered.rs
+++ b/tests/buffered.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind()
use tokio::net::TcpListener;
use tokio_test::assert_ok;
@@ -18,10 +18,10 @@
let msg = "foo bar baz";
let t = thread::spawn(move || {
- let mut s = assert_ok!(TcpStream::connect(&addr));
+ let mut s = assert_ok!(TcpStream::connect(addr));
let t2 = thread::spawn(move || {
- let mut s = assert_ok!(TcpStream::connect(&addr));
+ let mut s = assert_ok!(TcpStream::connect(addr));
let mut b = vec![0; msg.len() * N];
assert_ok!(s.read_exact(&mut b));
b
diff --git a/tests/fs.rs b/tests/fs.rs
index 13c44c0..ba38b71 100644
--- a/tests/fs.rs
+++ b/tests/fs.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations
use tokio::fs;
use tokio_test::assert_ok;
diff --git a/tests/fs_copy.rs b/tests/fs_copy.rs
index 8d16320..04678cf 100644
--- a/tests/fs_copy.rs
+++ b/tests/fs_copy.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations
use tempfile::tempdir;
use tokio::fs;
diff --git a/tests/fs_dir.rs b/tests/fs_dir.rs
index 21efe8c..f197a40 100644
--- a/tests/fs_dir.rs
+++ b/tests/fs_dir.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support directory operations
use tokio::fs;
use tokio_test::{assert_err, assert_ok};
diff --git a/tests/fs_file.rs b/tests/fs_file.rs
index f645e61..603ccad 100644
--- a/tests/fs_file.rs
+++ b/tests/fs_file.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations
use std::io::prelude::*;
use tempfile::NamedTempFile;
@@ -73,7 +73,7 @@
let mut buf = [0; 1024];
loop {
- file.read(&mut buf).await.unwrap();
+ let _ = file.read(&mut buf).await.unwrap();
file.seek(std::io::SeekFrom::Start(0)).await.unwrap();
}
});
diff --git a/tests/fs_link.rs b/tests/fs_link.rs
index 2ef666f..d198abc 100644
--- a/tests/fs_link.rs
+++ b/tests/fs_link.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations
use tokio::fs;
diff --git a/tests/io_buf_writer.rs b/tests/io_buf_writer.rs
index 47a0d46..d3acf62 100644
--- a/tests/io_buf_writer.rs
+++ b/tests/io_buf_writer.rs
@@ -70,15 +70,15 @@
async fn buf_writer() {
let mut writer = BufWriter::with_capacity(2, Vec::new());
- writer.write(&[0, 1]).await.unwrap();
+ assert_eq!(writer.write(&[0, 1]).await.unwrap(), 2);
assert_eq!(writer.buffer(), []);
assert_eq!(*writer.get_ref(), [0, 1]);
- writer.write(&[2]).await.unwrap();
+ assert_eq!(writer.write(&[2]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [2]);
assert_eq!(*writer.get_ref(), [0, 1]);
- writer.write(&[3]).await.unwrap();
+ assert_eq!(writer.write(&[3]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [2, 3]);
assert_eq!(*writer.get_ref(), [0, 1]);
@@ -86,20 +86,20 @@
assert_eq!(writer.buffer(), []);
assert_eq!(*writer.get_ref(), [0, 1, 2, 3]);
- writer.write(&[4]).await.unwrap();
- writer.write(&[5]).await.unwrap();
+ assert_eq!(writer.write(&[4]).await.unwrap(), 1);
+ assert_eq!(writer.write(&[5]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [4, 5]);
assert_eq!(*writer.get_ref(), [0, 1, 2, 3]);
- writer.write(&[6]).await.unwrap();
+ assert_eq!(writer.write(&[6]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [6]);
assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5]);
- writer.write(&[7, 8]).await.unwrap();
+ assert_eq!(writer.write(&[7, 8]).await.unwrap(), 2);
assert_eq!(writer.buffer(), []);
assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8]);
- writer.write(&[9, 10, 11]).await.unwrap();
+ assert_eq!(writer.write(&[9, 10, 11]).await.unwrap(), 3);
assert_eq!(writer.buffer(), []);
assert_eq!(*writer.get_ref(), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]);
@@ -111,7 +111,7 @@
#[tokio::test]
async fn buf_writer_inner_flushes() {
let mut w = BufWriter::with_capacity(3, Vec::new());
- w.write(&[0, 1]).await.unwrap();
+ assert_eq!(w.write(&[0, 1]).await.unwrap(), 2);
assert_eq!(*w.get_ref(), []);
w.flush().await.unwrap();
let w = w.into_inner();
@@ -135,15 +135,15 @@
async fn maybe_pending_buf_writer() {
let mut writer = BufWriter::with_capacity(2, MaybePending::new(Vec::new()));
- writer.write(&[0, 1]).await.unwrap();
+ assert_eq!(writer.write(&[0, 1]).await.unwrap(), 2);
assert_eq!(writer.buffer(), []);
assert_eq!(&writer.get_ref().inner, &[0, 1]);
- writer.write(&[2]).await.unwrap();
+ assert_eq!(writer.write(&[2]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [2]);
assert_eq!(&writer.get_ref().inner, &[0, 1]);
- writer.write(&[3]).await.unwrap();
+ assert_eq!(writer.write(&[3]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [2, 3]);
assert_eq!(&writer.get_ref().inner, &[0, 1]);
@@ -151,20 +151,20 @@
assert_eq!(writer.buffer(), []);
assert_eq!(&writer.get_ref().inner, &[0, 1, 2, 3]);
- writer.write(&[4]).await.unwrap();
- writer.write(&[5]).await.unwrap();
+ assert_eq!(writer.write(&[4]).await.unwrap(), 1);
+ assert_eq!(writer.write(&[5]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [4, 5]);
assert_eq!(&writer.get_ref().inner, &[0, 1, 2, 3]);
- writer.write(&[6]).await.unwrap();
+ assert_eq!(writer.write(&[6]).await.unwrap(), 1);
assert_eq!(writer.buffer(), [6]);
assert_eq!(writer.get_ref().inner, &[0, 1, 2, 3, 4, 5]);
- writer.write(&[7, 8]).await.unwrap();
+ assert_eq!(writer.write(&[7, 8]).await.unwrap(), 2);
assert_eq!(writer.buffer(), []);
assert_eq!(writer.get_ref().inner, &[0, 1, 2, 3, 4, 5, 6, 7, 8]);
- writer.write(&[9, 10, 11]).await.unwrap();
+ assert_eq!(writer.write(&[9, 10, 11]).await.unwrap(), 3);
assert_eq!(writer.buffer(), []);
assert_eq!(
writer.get_ref().inner,
@@ -182,7 +182,7 @@
#[tokio::test]
async fn maybe_pending_buf_writer_inner_flushes() {
let mut w = BufWriter::with_capacity(3, MaybePending::new(Vec::new()));
- w.write(&[0, 1]).await.unwrap();
+ assert_eq!(w.write(&[0, 1]).await.unwrap(), 2);
assert_eq!(&w.get_ref().inner, &[]);
w.flush().await.unwrap();
let w = w.into_inner().inner;
diff --git a/tests/io_copy_bidirectional.rs b/tests/io_copy_bidirectional.rs
index 0e82b29..c549675 100644
--- a/tests/io_copy_bidirectional.rs
+++ b/tests/io_copy_bidirectional.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind()
use std::time::Duration;
use tokio::io::{self, copy_bidirectional, AsyncReadExt, AsyncWriteExt};
@@ -111,18 +111,30 @@
}
#[tokio::test]
-async fn immediate_exit_on_error() {
- symmetric(|handle, mut a, mut b| async move {
- block_write(&mut a).await;
+async fn immediate_exit_on_write_error() {
+ let payload = b"here, take this";
+ let error = || io::Error::new(io::ErrorKind::Other, "no thanks!");
- // Fill up the b->copy->a path. We expect that this will _not_ drain
- // before we exit the copy task.
- let _bytes_written = block_write(&mut b).await;
+ let mut a = tokio_test::io::Builder::new()
+ .read(payload)
+ .write_error(error())
+ .build();
- // Drop b. We should not wait for a to consume the data buffered in the
- // copy loop, since b will be failing writes.
- drop(b);
- assert!(handle.await.unwrap().is_err());
- })
- .await
+ let mut b = tokio_test::io::Builder::new()
+ .read(payload)
+ .write_error(error())
+ .build();
+
+ assert!(copy_bidirectional(&mut a, &mut b).await.is_err());
+}
+
+#[tokio::test]
+async fn immediate_exit_on_read_error() {
+ let error = || io::Error::new(io::ErrorKind::Other, "got nothing!");
+
+ let mut a = tokio_test::io::Builder::new().read_error(error()).build();
+
+ let mut b = tokio_test::io::Builder::new().read_error(error()).build();
+
+ assert!(copy_bidirectional(&mut a, &mut b).await.is_err());
}
diff --git a/tests/io_driver.rs b/tests/io_driver.rs
index 6fb566d..97018e0 100644
--- a/tests/io_driver.rs
+++ b/tests/io_driver.rs
@@ -1,5 +1,6 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+// Wasi does not support panic recovery or threading
+#![cfg(all(feature = "full", not(tokio_wasi)))]
use tokio::net::TcpListener;
use tokio::runtime;
@@ -79,7 +80,7 @@
drop(task);
// Establish a connection to the acceptor
- let _s = TcpStream::connect(&addr).unwrap();
+ let _s = TcpStream::connect(addr).unwrap();
// Force the reactor to turn
rt.block_on(async {});
diff --git a/tests/io_driver_drop.rs b/tests/io_driver_drop.rs
index 631e66e..0a384d4 100644
--- a/tests/io_driver_drop.rs
+++ b/tests/io_driver_drop.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind
use tokio::net::TcpListener;
use tokio::runtime;
diff --git a/tests/io_fill_buf.rs b/tests/io_fill_buf.rs
index 0b2ebd7..62c3f1b 100644
--- a/tests/io_fill_buf.rs
+++ b/tests/io_fill_buf.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support file operations
use tempfile::NamedTempFile;
use tokio::fs::File;
diff --git a/tests/io_mem_stream.rs b/tests/io_mem_stream.rs
index 520391a..3d9941d 100644
--- a/tests/io_mem_stream.rs
+++ b/tests/io_mem_stream.rs
@@ -101,3 +101,22 @@
// drop b only after task t1 finishes writing
drop(b);
}
+
+#[tokio::test]
+async fn duplex_is_cooperative() {
+ let (mut tx, mut rx) = tokio::io::duplex(1024 * 8);
+
+ tokio::select! {
+ biased;
+
+ _ = async {
+ loop {
+ let buf = [3u8; 4096];
+ tx.write_all(&buf).await.unwrap();
+ let mut buf = [0u8; 4096];
+ let _ = rx.read(&mut buf).await.unwrap();
+ }
+ } => {},
+ _ = tokio::task::yield_now() => {}
+ }
+}
diff --git a/tests/io_panic.rs b/tests/io_panic.rs
new file mode 100644
index 0000000..922d7c0
--- /dev/null
+++ b/tests/io_panic.rs
@@ -0,0 +1,176 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
+
+use std::task::{Context, Poll};
+use std::{error::Error, pin::Pin};
+use tokio::io::{self, split, AsyncRead, AsyncWrite, ReadBuf};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+struct RW;
+
+impl AsyncRead for RW {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ _cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<io::Result<()>> {
+ buf.put_slice(&[b'z']);
+ Poll::Ready(Ok(()))
+ }
+}
+
+impl AsyncWrite for RW {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ _cx: &mut Context<'_>,
+ _buf: &[u8],
+ ) -> Poll<Result<usize, io::Error>> {
+ Poll::Ready(Ok(1))
+ }
+
+ fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
+ Poll::Ready(Ok(()))
+ }
+}
+
+#[cfg(unix)]
+mod unix {
+ use std::os::unix::prelude::{AsRawFd, RawFd};
+
+ pub struct MockFd;
+
+ impl AsRawFd for MockFd {
+ fn as_raw_fd(&self) -> RawFd {
+ 0
+ }
+ }
+}
+
+#[test]
+fn read_buf_initialize_unfilled_to_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let mut buffer = Vec::<u8>::new();
+ let mut read_buf = ReadBuf::new(&mut buffer);
+
+ read_buf.initialize_unfilled_to(2);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn read_buf_advance_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let mut buffer = Vec::<u8>::new();
+ let mut read_buf = ReadBuf::new(&mut buffer);
+
+ read_buf.advance(2);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn read_buf_set_filled_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let mut buffer = Vec::<u8>::new();
+ let mut read_buf = ReadBuf::new(&mut buffer);
+
+ read_buf.set_filled(2);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn read_buf_put_slice_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let mut buffer = Vec::<u8>::new();
+ let mut read_buf = ReadBuf::new(&mut buffer);
+
+ let new_slice = [0x40_u8, 0x41_u8];
+
+ read_buf.put_slice(&new_slice);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn unsplit_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let (r1, _w1) = split(RW);
+ let (_r2, w2) = split(RW);
+ r1.unsplit(w2);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(unix)]
+fn async_fd_new_panic_caller() -> Result<(), Box<dyn Error>> {
+ use tokio::io::unix::AsyncFd;
+ use tokio::runtime::Builder;
+
+ let panic_location_file = test_panic(|| {
+ // Runtime without `enable_io` so it has no IO driver set.
+ let rt = Builder::new_current_thread().build().unwrap();
+ rt.block_on(async {
+ let fd = unix::MockFd;
+
+ let _ = AsyncFd::new(fd);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(unix)]
+fn async_fd_with_interest_panic_caller() -> Result<(), Box<dyn Error>> {
+ use tokio::io::unix::AsyncFd;
+ use tokio::io::Interest;
+ use tokio::runtime::Builder;
+
+ let panic_location_file = test_panic(|| {
+ // Runtime without `enable_io` so it has no IO driver set.
+ let rt = Builder::new_current_thread().build().unwrap();
+ rt.block_on(async {
+ let fd = unix::MockFd;
+
+ let _ = AsyncFd::with_interest(fd, Interest::READABLE);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
diff --git a/tests/io_read.rs b/tests/io_read.rs
index cb1aa70..7f74c5e 100644
--- a/tests/io_read.rs
+++ b/tests/io_read.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
use tokio_test::assert_ok;
@@ -8,6 +8,11 @@
use std::pin::Pin;
use std::task::{Context, Poll};
+mod support {
+ pub(crate) mod leaked_buffers;
+}
+use support::leaked_buffers::LeakedBuffers;
+
#[tokio::test]
async fn read() {
#[derive(Default)]
@@ -37,16 +42,27 @@
assert_eq!(buf[..], b"hello world"[..]);
}
-struct BadAsyncRead;
+struct BadAsyncRead {
+ leaked_buffers: LeakedBuffers,
+}
+
+impl BadAsyncRead {
+ fn new() -> Self {
+ Self {
+ leaked_buffers: LeakedBuffers::new(),
+ }
+ }
+}
impl AsyncRead for BadAsyncRead {
fn poll_read(
- self: Pin<&mut Self>,
+ mut self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
- *buf = ReadBuf::new(Box::leak(vec![0; buf.capacity()].into_boxed_slice()));
+ *buf = ReadBuf::new(unsafe { self.leaked_buffers.create(buf.capacity()) });
buf.advance(buf.capacity());
+
Poll::Ready(Ok(()))
}
}
@@ -55,5 +71,5 @@
#[should_panic]
async fn read_buf_bad_async_read() {
let mut buf = Vec::with_capacity(10);
- BadAsyncRead.read_buf(&mut buf).await.unwrap();
+ BadAsyncRead::new().read_buf(&mut buf).await.unwrap();
}
diff --git a/tests/io_split.rs b/tests/io_split.rs
index a012166..4cbd2a2 100644
--- a/tests/io_split.rs
+++ b/tests/io_split.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
use tokio::io::{split, AsyncRead, AsyncWrite, ReadBuf, ReadHalf, WriteHalf};
diff --git a/tests/io_take.rs b/tests/io_take.rs
index 683606f..fba01d2 100644
--- a/tests/io_take.rs
+++ b/tests/io_take.rs
@@ -1,9 +1,16 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
-use tokio::io::AsyncReadExt;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::io::{self, AsyncRead, AsyncReadExt, ReadBuf};
use tokio_test::assert_ok;
+mod support {
+ pub(crate) mod leaked_buffers;
+}
+use support::leaked_buffers::LeakedBuffers;
+
#[tokio::test]
async fn take() {
let mut buf = [0; 6];
@@ -14,3 +21,54 @@
assert_eq!(n, 4);
assert_eq!(&buf, &b"hell\0\0"[..]);
}
+
+#[tokio::test]
+async fn issue_4435() {
+ let mut buf = [0; 8];
+ let rd: &[u8] = b"hello world";
+
+ let rd = rd.take(4);
+ tokio::pin!(rd);
+
+ let mut read_buf = ReadBuf::new(&mut buf);
+ read_buf.put_slice(b"AB");
+
+ futures::future::poll_fn(|cx| rd.as_mut().poll_read(cx, &mut read_buf))
+ .await
+ .unwrap();
+ assert_eq!(&buf, &b"ABhell\0\0"[..]);
+}
+
+struct BadReader {
+ leaked_buffers: LeakedBuffers,
+}
+
+impl BadReader {
+ fn new() -> Self {
+ Self {
+ leaked_buffers: LeakedBuffers::new(),
+ }
+ }
+}
+
+impl AsyncRead for BadReader {
+ fn poll_read(
+ mut self: Pin<&mut Self>,
+ _cx: &mut Context<'_>,
+ read_buf: &mut ReadBuf<'_>,
+ ) -> Poll<io::Result<()>> {
+ let mut buf = ReadBuf::new(unsafe { self.leaked_buffers.create(10) });
+ buf.put_slice(&[123; 10]);
+ *read_buf = buf;
+
+ Poll::Ready(Ok(()))
+ }
+}
+
+#[tokio::test]
+#[should_panic]
+async fn bad_reader_fails() {
+ let mut buf = Vec::with_capacity(10);
+
+ BadReader::new().take(10).read_buf(&mut buf).await.unwrap();
+}
diff --git a/tests/io_util_empty.rs b/tests/io_util_empty.rs
new file mode 100644
index 0000000..e49cd17
--- /dev/null
+++ b/tests/io_util_empty.rs
@@ -0,0 +1,32 @@
+#![cfg(feature = "full")]
+use tokio::io::{AsyncBufReadExt, AsyncReadExt};
+
+#[tokio::test]
+async fn empty_read_is_cooperative() {
+ tokio::select! {
+ biased;
+
+ _ = async {
+ loop {
+ let mut buf = [0u8; 4096];
+ let _ = tokio::io::empty().read(&mut buf).await;
+ }
+ } => {},
+ _ = tokio::task::yield_now() => {}
+ }
+}
+
+#[tokio::test]
+async fn empty_buf_reads_are_cooperative() {
+ tokio::select! {
+ biased;
+
+ _ = async {
+ loop {
+ let mut buf = String::new();
+ let _ = tokio::io::empty().read_line(&mut buf).await;
+ }
+ } => {},
+ _ = tokio::task::yield_now() => {}
+ }
+}
diff --git a/tests/join_handle_panic.rs b/tests/join_handle_panic.rs
new file mode 100644
index 0000000..bc34fd0
--- /dev/null
+++ b/tests/join_handle_panic.rs
@@ -0,0 +1,20 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
+
+struct PanicsOnDrop;
+
+impl Drop for PanicsOnDrop {
+ fn drop(&mut self) {
+ panic!("I told you so");
+ }
+}
+
+#[tokio::test]
+async fn test_panics_do_not_propagate_when_dropping_join_handle() {
+ let join_handle = tokio::spawn(async move { PanicsOnDrop });
+
+ // only drop the JoinHandle when the task has completed
+ // (which is difficult to synchronize precisely)
+ tokio::time::sleep(std::time::Duration::from_millis(3)).await;
+ drop(join_handle);
+}
diff --git a/tests/macros_join.rs b/tests/macros_join.rs
index 169e898..9e4d234 100644
--- a/tests/macros_join.rs
+++ b/tests/macros_join.rs
@@ -1,36 +1,48 @@
-#![allow(clippy::blacklisted_name)]
-use tokio::sync::oneshot;
+#![cfg(feature = "macros")]
+#![allow(clippy::disallowed_names)]
+use std::sync::Arc;
+
+#[cfg(tokio_wasm_not_wasi)]
+#[cfg(target_pointer_width = "64")]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
+
+use tokio::sync::{oneshot, Semaphore};
use tokio_test::{assert_pending, assert_ready, task};
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_lit_expr_comma() {
let foo = tokio::join!(async { 1 },);
assert_eq!(foo, (1,));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_lit_expr_no_comma() {
let foo = tokio::join!(async { 1 });
assert_eq!(foo, (1,));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_two_lit_expr_comma() {
let foo = tokio::join!(async { 1 }, async { 2 },);
assert_eq!(foo, (1, 2));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_two_lit_expr_no_comma() {
let foo = tokio::join!(async { 1 }, async { 2 });
assert_eq!(foo, (1, 2));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn two_await() {
let (tx1, rx1) = oneshot::channel::<&str>();
let (tx2, rx2) = oneshot::channel::<u32>();
@@ -53,6 +65,8 @@
}
#[test]
+#[cfg(target_pointer_width = "64")]
+#[ignore = "Android: ignore until the compiler is updated. aliceryhl@ says these tests assume latest stable compiler."]
fn join_size() {
use futures::future;
use std::mem;
@@ -61,12 +75,82 @@
let ready = future::ready(0i32);
tokio::join!(ready)
};
- assert_eq!(mem::size_of_val(&fut), 16);
+ assert_eq!(mem::size_of_val(&fut), 32);
let fut = async {
let ready1 = future::ready(0i32);
let ready2 = future::ready(0i32);
tokio::join!(ready1, ready2)
};
- assert_eq!(mem::size_of_val(&fut), 28);
+ assert_eq!(mem::size_of_val(&fut), 40);
+}
+
+async fn non_cooperative_task(permits: Arc<Semaphore>) -> usize {
+ let mut exceeded_budget = 0;
+
+ for _ in 0..5 {
+ // Another task should run after this task uses its whole budget
+ for _ in 0..128 {
+ let _permit = permits.clone().acquire_owned().await.unwrap();
+ }
+
+ exceeded_budget += 1;
+ }
+
+ exceeded_budget
+}
+
+async fn poor_little_task(permits: Arc<Semaphore>) -> usize {
+ let mut how_many_times_i_got_to_run = 0;
+
+ for _ in 0..5 {
+ let _permit = permits.clone().acquire_owned().await.unwrap();
+ how_many_times_i_got_to_run += 1;
+ }
+
+ how_many_times_i_got_to_run
+}
+
+#[tokio::test]
+async fn join_does_not_allow_tasks_to_starve() {
+ let permits = Arc::new(Semaphore::new(1));
+
+ // non_cooperative_task should yield after its budget is exceeded and then poor_little_task should run.
+ let (non_cooperative_result, little_task_result) = tokio::join!(
+ non_cooperative_task(Arc::clone(&permits)),
+ poor_little_task(permits)
+ );
+
+ assert_eq!(5, non_cooperative_result);
+ assert_eq!(5, little_task_result);
+}
+
+#[tokio::test]
+async fn a_different_future_is_polled_first_every_time_poll_fn_is_polled() {
+ let poll_order = Arc::new(std::sync::Mutex::new(vec![]));
+
+ let fut = |x, poll_order: Arc<std::sync::Mutex<Vec<i32>>>| async move {
+ for _ in 0..4 {
+ {
+ let mut guard = poll_order.lock().unwrap();
+
+ guard.push(x);
+ }
+
+ tokio::task::yield_now().await;
+ }
+ };
+
+ tokio::join!(
+ fut(1, Arc::clone(&poll_order)),
+ fut(2, Arc::clone(&poll_order)),
+ fut(3, Arc::clone(&poll_order)),
+ );
+
+ // Each time the future created by join! is polled, it should start
+ // by polling a different future first.
+ assert_eq!(
+ vec![1, 2, 3, 2, 3, 1, 3, 1, 2, 1, 2, 3],
+ *poll_order.lock().unwrap()
+ );
}
diff --git a/tests/macros_pin.rs b/tests/macros_pin.rs
index da6e0be..e99a3ee 100644
--- a/tests/macros_pin.rs
+++ b/tests/macros_pin.rs
@@ -1,7 +1,15 @@
+#![cfg(feature = "macros")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
+
async fn one() {}
async fn two() {}
-#[tokio::test]
+#[maybe_tokio_test]
async fn multi_pin() {
tokio::pin! {
let f1 = one();
diff --git a/tests/macros_rename_test.rs b/tests/macros_rename_test.rs
new file mode 100644
index 0000000..90a86f9
--- /dev/null
+++ b/tests/macros_rename_test.rs
@@ -0,0 +1,26 @@
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threading
+
+#[allow(unused_imports)]
+use std as tokio;
+
+use ::tokio as tokio1;
+
+async fn compute() -> usize {
+ let join = tokio1::spawn(async { 1 });
+ join.await.unwrap()
+}
+
+#[tokio1::main(crate = "tokio1")]
+async fn compute_main() -> usize {
+ compute().await
+}
+
+#[test]
+fn crate_rename_main() {
+ assert_eq!(1, compute_main());
+}
+
+#[tokio1::test(crate = "tokio1")]
+async fn crate_rename_test() {
+ assert_eq!(1, compute().await);
+}
diff --git a/tests/macros_select.rs b/tests/macros_select.rs
index b4f8544..26d6fec 100644
--- a/tests/macros_select.rs
+++ b/tests/macros_select.rs
@@ -1,12 +1,19 @@
-#![allow(clippy::blacklisted_name)]
-use tokio::sync::{mpsc, oneshot};
-use tokio::task;
+#![cfg(feature = "macros")]
+#![allow(clippy::disallowed_names)]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
+
+use tokio::sync::oneshot;
use tokio_test::{assert_ok, assert_pending, assert_ready};
use futures::future::poll_fn;
use std::task::Poll::Ready;
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_lit_expr_comma() {
let foo = tokio::select! {
foo = async { 1 } => foo,
@@ -15,7 +22,7 @@
assert_eq!(foo, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn nested_one() {
let foo = tokio::select! {
foo = async { 1 } => tokio::select! {
@@ -26,7 +33,7 @@
assert_eq!(foo, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_lit_expr_no_comma() {
let foo = tokio::select! {
foo = async { 1 } => foo
@@ -35,7 +42,7 @@
assert_eq!(foo, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_lit_expr_block() {
let foo = tokio::select! {
foo = async { 1 } => { foo }
@@ -44,7 +51,7 @@
assert_eq!(foo, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_await() {
let foo = tokio::select! {
foo = one() => foo,
@@ -53,7 +60,7 @@
assert_eq!(foo, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_ident() {
let one = one();
@@ -64,7 +71,7 @@
assert_eq!(foo, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_two() {
use std::cell::Cell;
@@ -85,7 +92,7 @@
assert!(res == 1 || res == 2);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn drop_in_fut() {
let s = "hello".to_string();
@@ -100,7 +107,8 @@
assert_eq!(res, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
+#[cfg(feature = "full")]
async fn one_ready() {
let (tx1, rx1) = oneshot::channel::<i32>();
let (_tx2, rx2) = oneshot::channel::<i32>();
@@ -117,20 +125,23 @@
assert_eq!(1, v);
}
-#[tokio::test]
+#[maybe_tokio_test]
+#[cfg(feature = "full")]
async fn select_streams() {
+ use tokio::sync::mpsc;
+
let (tx1, mut rx1) = mpsc::unbounded_channel::<i32>();
let (tx2, mut rx2) = mpsc::unbounded_channel::<i32>();
tokio::spawn(async move {
assert_ok!(tx2.send(1));
- task::yield_now().await;
+ tokio::task::yield_now().await;
assert_ok!(tx1.send(2));
- task::yield_now().await;
+ tokio::task::yield_now().await;
assert_ok!(tx2.send(3));
- task::yield_now().await;
+ tokio::task::yield_now().await;
drop((tx1, tx2));
});
@@ -156,7 +167,7 @@
assert_eq!(&msgs[..], &[1, 2, 3]);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn move_uncompleted_futures() {
let (tx1, mut rx1) = oneshot::channel::<i32>();
let (tx2, mut rx2) = oneshot::channel::<i32>();
@@ -182,7 +193,7 @@
assert!(ran);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn nested() {
let res = tokio::select! {
x = async { 1 } => {
@@ -195,7 +206,8 @@
assert_eq!(res, 3);
}
-#[tokio::test]
+#[maybe_tokio_test]
+#[cfg(target_pointer_width = "64")]
async fn struct_size() {
use futures::future;
use std::mem;
@@ -208,7 +220,7 @@
}
};
- assert!(mem::size_of_val(&fut) <= 32);
+ assert_eq!(mem::size_of_val(&fut), 40);
let fut = async {
let ready1 = future::ready(0i32);
@@ -220,7 +232,7 @@
}
};
- assert!(mem::size_of_val(&fut) <= 40);
+ assert_eq!(mem::size_of_val(&fut), 48);
let fut = async {
let ready1 = future::ready(0i32);
@@ -234,10 +246,10 @@
}
};
- assert!(mem::size_of_val(&fut) <= 48);
+ assert_eq!(mem::size_of_val(&fut), 56);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn mutable_borrowing_future_with_same_borrow_in_block() {
let mut value = 234;
@@ -251,7 +263,7 @@
assert!(value >= 234);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn mutable_borrowing_future_with_same_borrow_in_block_and_else() {
let mut value = 234;
@@ -268,7 +280,7 @@
assert!(value >= 234);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn future_panics_after_poll() {
use tokio_test::task;
@@ -298,7 +310,7 @@
assert_eq!(1, res);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn disable_with_if() {
use tokio_test::task;
@@ -320,7 +332,7 @@
assert_ready!(f.poll());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn join_with_select() {
use tokio_test::task;
@@ -356,6 +368,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn use_future_in_if_condition() {
use tokio::time::{self, Duration};
@@ -369,6 +382,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn use_future_in_if_condition_biased() {
use tokio::time::{self, Duration};
@@ -382,7 +396,7 @@
}
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn many_branches() {
let num = tokio::select! {
x = async { 1 } => x,
@@ -448,12 +462,13 @@
x = async { 1 } => x,
x = async { 1 } => x,
x = async { 1 } => x,
+ x = async { 1 } => x,
};
assert_eq!(1, num);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn never_branch_no_warnings() {
let t = tokio::select! {
_ = async_never() => 0,
@@ -474,7 +489,7 @@
}
// From https://github.com/tokio-rs/tokio/issues/2857
-#[tokio::test]
+#[maybe_tokio_test]
async fn mut_on_left_hand_side() {
let v = async move {
let ok = async { 1 };
@@ -490,7 +505,7 @@
assert_eq!(v, 2);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn biased_one_not_ready() {
let (_tx1, rx1) = oneshot::channel::<i32>();
let (tx2, rx2) = oneshot::channel::<i32>();
@@ -514,7 +529,8 @@
assert_eq!(2, v);
}
-#[tokio::test]
+#[maybe_tokio_test]
+#[cfg(feature = "full")]
async fn biased_eventually_ready() {
use tokio::task::yield_now;
@@ -560,7 +576,7 @@
}
// https://github.com/tokio-rs/tokio/issues/4182
-#[tokio::test]
+#[maybe_tokio_test]
async fn mut_ref_patterns() {
tokio::select! {
Some(mut foo) = async { Some("1".to_string()) } => {
@@ -584,3 +600,66 @@
},
};
}
+
+#[cfg(tokio_unstable)]
+mod unstable {
+ use tokio::runtime::RngSeed;
+
+ #[test]
+ fn deterministic_select_current_thread() {
+ let seed = b"bytes used to generate seed";
+ let rt1 = tokio::runtime::Builder::new_current_thread()
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt1_values = rt1.block_on(async { (select_0_to_9().await, select_0_to_9().await) });
+
+ let rt2 = tokio::runtime::Builder::new_current_thread()
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt2_values = rt2.block_on(async { (select_0_to_9().await, select_0_to_9().await) });
+
+ assert_eq!(rt1_values, rt2_values);
+ }
+
+ #[test]
+ #[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))]
+ fn deterministic_select_multi_thread() {
+ let seed = b"bytes used to generate seed";
+ let rt1 = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt1_values = rt1.block_on(async {
+ let _ = tokio::spawn(async { (select_0_to_9().await, select_0_to_9().await) }).await;
+ });
+
+ let rt2 = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt2_values = rt2.block_on(async {
+ let _ = tokio::spawn(async { (select_0_to_9().await, select_0_to_9().await) }).await;
+ });
+
+ assert_eq!(rt1_values, rt2_values);
+ }
+
+ async fn select_0_to_9() -> u32 {
+ tokio::select!(
+ x = async { 0 } => x,
+ x = async { 1 } => x,
+ x = async { 2 } => x,
+ x = async { 3 } => x,
+ x = async { 4 } => x,
+ x = async { 5 } => x,
+ x = async { 6 } => x,
+ x = async { 7 } => x,
+ x = async { 8 } => x,
+ x = async { 9 } => x,
+ )
+ }
+}
diff --git a/tests/macros_test.rs b/tests/macros_test.rs
index bca2c91..85279b7 100644
--- a/tests/macros_test.rs
+++ b/tests/macros_test.rs
@@ -1,3 +1,5 @@
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threading
+
use tokio::test;
#[test]
@@ -46,3 +48,41 @@
return Ok(());
panic!();
}
+
+// https://github.com/tokio-rs/tokio/issues/4175
+#[allow(clippy::let_unit_value)]
+pub mod clippy_semicolon_if_nothing_returned {
+ #![deny(clippy::semicolon_if_nothing_returned)]
+
+ #[tokio::main]
+ pub async fn local() {
+ let _x = ();
+ }
+ #[tokio::main]
+ pub async fn item() {
+ fn _f() {}
+ }
+ #[tokio::main]
+ pub async fn semi() {
+ panic!();
+ }
+ #[tokio::main]
+ pub async fn empty() {
+ // To trigger clippy::semicolon_if_nothing_returned lint, the block needs to contain newline.
+ }
+}
+
+// https://github.com/tokio-rs/tokio/issues/5243
+pub mod issue_5243 {
+ macro_rules! mac {
+ (async fn $name:ident() $b:block) => {
+ #[::tokio::test]
+ async fn $name() {
+ $b
+ }
+ };
+ }
+ mac!(
+ async fn foo() {}
+ );
+}
diff --git a/tests/macros_try_join.rs b/tests/macros_try_join.rs
index a925153..6c43222 100644
--- a/tests/macros_try_join.rs
+++ b/tests/macros_try_join.rs
@@ -1,36 +1,46 @@
-#![allow(clippy::blacklisted_name)]
-use tokio::sync::oneshot;
+#![cfg(feature = "macros")]
+#![allow(clippy::disallowed_names)]
+
+use std::sync::Arc;
+
+use tokio::sync::{oneshot, Semaphore};
use tokio_test::{assert_pending, assert_ready, task};
-#[tokio::test]
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
+
+#[maybe_tokio_test]
async fn sync_one_lit_expr_comma() {
let foo = tokio::try_join!(async { ok(1) },);
assert_eq!(foo, Ok((1,)));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_one_lit_expr_no_comma() {
let foo = tokio::try_join!(async { ok(1) });
assert_eq!(foo, Ok((1,)));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_two_lit_expr_comma() {
let foo = tokio::try_join!(async { ok(1) }, async { ok(2) },);
assert_eq!(foo, Ok((1, 2)));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn sync_two_lit_expr_no_comma() {
let foo = tokio::try_join!(async { ok(1) }, async { ok(2) });
assert_eq!(foo, Ok((1, 2)));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn two_await() {
let (tx1, rx1) = oneshot::channel::<&str>();
let (tx2, rx2) = oneshot::channel::<u32>();
@@ -51,7 +61,7 @@
assert_eq!(Ok(("hello", 123)), res);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn err_abort_early() {
let (tx1, rx1) = oneshot::channel::<&str>();
let (tx2, rx2) = oneshot::channel::<u32>();
@@ -78,6 +88,7 @@
}
#[test]
+#[cfg(target_pointer_width = "64")]
fn join_size() {
use futures::future;
use std::mem;
@@ -86,16 +97,89 @@
let ready = future::ready(ok(0i32));
tokio::try_join!(ready)
};
- assert_eq!(mem::size_of_val(&fut), 16);
+ assert_eq!(mem::size_of_val(&fut), 32);
let fut = async {
let ready1 = future::ready(ok(0i32));
let ready2 = future::ready(ok(0i32));
tokio::try_join!(ready1, ready2)
};
- assert_eq!(mem::size_of_val(&fut), 28);
+ assert_eq!(mem::size_of_val(&fut), 48);
}
fn ok<T>(val: T) -> Result<T, ()> {
Ok(val)
}
+
+async fn non_cooperative_task(permits: Arc<Semaphore>) -> Result<usize, String> {
+ let mut exceeded_budget = 0;
+
+ for _ in 0..5 {
+ // Another task should run after this task uses its whole budget
+ for _ in 0..128 {
+ let _permit = permits.clone().acquire_owned().await.unwrap();
+ }
+
+ exceeded_budget += 1;
+ }
+
+ Ok(exceeded_budget)
+}
+
+async fn poor_little_task(permits: Arc<Semaphore>) -> Result<usize, String> {
+ let mut how_many_times_i_got_to_run = 0;
+
+ for _ in 0..5 {
+ let _permit = permits.clone().acquire_owned().await.unwrap();
+
+ how_many_times_i_got_to_run += 1;
+ }
+
+ Ok(how_many_times_i_got_to_run)
+}
+
+#[tokio::test]
+async fn try_join_does_not_allow_tasks_to_starve() {
+ let permits = Arc::new(Semaphore::new(10));
+
+ // non_cooperative_task should yield after its budget is exceeded and then poor_little_task should run.
+ let result = tokio::try_join!(
+ non_cooperative_task(Arc::clone(&permits)),
+ poor_little_task(permits)
+ );
+
+ let (non_cooperative_result, little_task_result) = result.unwrap();
+
+ assert_eq!(5, non_cooperative_result);
+ assert_eq!(5, little_task_result);
+}
+
+#[tokio::test]
+async fn a_different_future_is_polled_first_every_time_poll_fn_is_polled() {
+ let poll_order = Arc::new(std::sync::Mutex::new(vec![]));
+
+ let fut = |x, poll_order: Arc<std::sync::Mutex<Vec<i32>>>| async move {
+ for _ in 0..4 {
+ {
+ let mut guard = poll_order.lock().unwrap();
+
+ guard.push(x);
+ }
+
+ tokio::task::yield_now().await;
+ }
+ };
+
+ tokio::join!(
+ fut(1, Arc::clone(&poll_order)),
+ fut(2, Arc::clone(&poll_order)),
+ fut(3, Arc::clone(&poll_order)),
+ );
+
+ // Each time the future created by join! is polled, it should start
+ // by polling a different future first.
+ assert_eq!(
+ vec![1, 2, 3, 2, 3, 1, 3, 1, 2, 1, 2, 3],
+ *poll_order.lock().unwrap()
+ );
+}
diff --git a/tests/net_bind_resource.rs b/tests/net_bind_resource.rs
index d4a0b8d..1c604aa 100644
--- a/tests/net_bind_resource.rs
+++ b/tests/net_bind_resource.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery or bind
use tokio::net::TcpListener;
diff --git a/tests/net_lookup_host.rs b/tests/net_lookup_host.rs
index 4d06402..2b346e6 100644
--- a/tests/net_lookup_host.rs
+++ b/tests/net_lookup_host.rs
@@ -1,3 +1,5 @@
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support direct socket operations
+
use tokio::net;
use tokio_test::assert_ok;
diff --git a/tests/named_pipe.rs b/tests/net_named_pipe.rs
similarity index 89%
rename from tests/named_pipe.rs
rename to tests/net_named_pipe.rs
index 2055c3c..c421224 100644
--- a/tests/named_pipe.rs
+++ b/tests/net_named_pipe.rs
@@ -8,7 +8,7 @@
use tokio::io::AsyncWriteExt;
use tokio::net::windows::named_pipe::{ClientOptions, PipeMode, ServerOptions};
use tokio::time;
-use winapi::shared::winerror;
+use windows_sys::Win32::Foundation::{ERROR_NO_DATA, ERROR_PIPE_BUSY, NO_ERROR, UNICODE_STRING};
#[tokio::test]
async fn test_named_pipe_client_drop() -> io::Result<()> {
@@ -25,7 +25,7 @@
// instance will be broken because client is gone
match server.write_all(b"ping").await {
- Err(e) if e.raw_os_error() == Some(winerror::ERROR_NO_DATA as i32) => (),
+ Err(e) if e.raw_os_error() == Some(ERROR_NO_DATA as i32) => (),
x => panic!("{:?}", x),
}
@@ -120,7 +120,7 @@
let client = loop {
match ClientOptions::new().open(PIPE_NAME) {
Ok(client) => break client,
- Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+ Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
Err(e) if e.kind() == io::ErrorKind::NotFound => (),
Err(e) => return Err(e),
}
@@ -249,7 +249,7 @@
let client = loop {
match ClientOptions::new().open(PIPE_NAME) {
Ok(client) => break client,
- Err(e) if e.raw_os_error() == Some(winerror::ERROR_PIPE_BUSY as i32) => (),
+ Err(e) if e.raw_os_error() == Some(ERROR_PIPE_BUSY as i32) => (),
Err(e) if e.kind() == io::ErrorKind::NotFound => (),
Err(e) => return Err(e),
}
@@ -341,12 +341,38 @@
Ok(())
}
+// This tests `NamedPipeServer::connect` with various access settings.
+#[tokio::test]
+async fn test_named_pipe_access() -> io::Result<()> {
+ const PIPE_NAME: &str = r"\\.\pipe\test-named-pipe-access";
+
+ for (inb, outb) in [(true, true), (true, false), (false, true)] {
+ let (tx, rx) = tokio::sync::oneshot::channel();
+ let server = tokio::spawn(async move {
+ let s = ServerOptions::new()
+ .access_inbound(inb)
+ .access_outbound(outb)
+ .create(PIPE_NAME)?;
+ let mut connect_fut = tokio_test::task::spawn(s.connect());
+ assert!(connect_fut.poll().is_pending());
+ tx.send(()).unwrap();
+ connect_fut.await
+ });
+
+ // Wait for the server to call connect.
+ rx.await.unwrap();
+ let _ = ClientOptions::new().read(outb).write(inb).open(PIPE_NAME)?;
+
+ server.await??;
+ }
+ Ok(())
+}
+
fn num_instances(pipe_name: impl AsRef<str>) -> io::Result<u32> {
use ntapi::ntioapi;
- use winapi::shared::ntdef;
let mut name = pipe_name.as_ref().encode_utf16().collect::<Vec<_>>();
- let mut name = ntdef::UNICODE_STRING {
+ let mut name = UNICODE_STRING {
Length: (name.len() * mem::size_of::<u16>()) as u16,
MaximumLength: (name.len() * mem::size_of::<u16>()) as u16,
Buffer: name.as_mut_ptr(),
@@ -366,12 +392,12 @@
1024,
ntioapi::FileDirectoryInformation,
0,
- &mut name,
+ &mut name as *mut _ as _,
0,
)
};
- if status as u32 != winerror::NO_ERROR {
+ if status as u32 != NO_ERROR {
return Err(io::Error::last_os_error());
}
diff --git a/tests/net_panic.rs b/tests/net_panic.rs
new file mode 100644
index 0000000..fc2209a
--- /dev/null
+++ b/tests/net_panic.rs
@@ -0,0 +1,188 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(tokio_wasi)))]
+
+use std::error::Error;
+use tokio::net::{TcpListener, TcpStream};
+use tokio::runtime::{Builder, Runtime};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+#[test]
+fn udp_socket_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
+ use std::net::SocketAddr;
+ use tokio::net::UdpSocket;
+
+ let addr = "127.0.0.1:0".parse::<SocketAddr>().unwrap();
+ let std_sock = std::net::UdpSocket::bind(addr).unwrap();
+ std_sock.set_nonblocking(true).unwrap();
+
+ let panic_location_file = test_panic(|| {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _sock = UdpSocket::from_std(std_sock);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn tcp_listener_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
+ let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ std_listener.set_nonblocking(true).unwrap();
+
+ let panic_location_file = test_panic(|| {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _ = TcpListener::from_std(std_listener);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn tcp_stream_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
+ let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+
+ let std_stream = std::net::TcpStream::connect(std_listener.local_addr().unwrap()).unwrap();
+ std_stream.set_nonblocking(true).unwrap();
+
+ let panic_location_file = test_panic(|| {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _ = TcpStream::from_std(std_stream);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(unix)]
+fn unix_listener_bind_panic_caller() -> Result<(), Box<dyn Error>> {
+ use tokio::net::UnixListener;
+
+ let dir = tempfile::tempdir().unwrap();
+ let sock_path = dir.path().join("socket");
+
+ let panic_location_file = test_panic(|| {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _ = UnixListener::bind(&sock_path);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(unix)]
+fn unix_listener_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
+ use tokio::net::UnixListener;
+
+ let dir = tempfile::tempdir().unwrap();
+ let sock_path = dir.path().join("socket");
+ let std_listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap();
+
+ let panic_location_file = test_panic(|| {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _ = UnixListener::from_std(std_listener);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(unix)]
+fn unix_stream_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
+ use tokio::net::UnixStream;
+
+ let dir = tempfile::tempdir().unwrap();
+ let sock_path = dir.path().join("socket");
+ let _std_listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap();
+ let std_stream = std::os::unix::net::UnixStream::connect(&sock_path).unwrap();
+
+ let panic_location_file = test_panic(|| {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _ = UnixStream::from_std(std_stream);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(unix)]
+fn unix_datagram_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
+ use std::os::unix::net::UnixDatagram as StdUDS;
+ use tokio::net::UnixDatagram;
+
+ let dir = tempfile::tempdir().unwrap();
+ let sock_path = dir.path().join("socket");
+
+ // Bind the socket to a filesystem path
+ // /let socket_path = tmp.path().join("socket");
+ let std_socket = StdUDS::bind(&sock_path).unwrap();
+ std_socket.set_nonblocking(true).unwrap();
+
+ let panic_location_file = test_panic(move || {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let _ = UnixDatagram::from_std(std_socket);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+#[cfg(windows)]
+fn server_options_max_instances_panic_caller() -> Result<(), Box<dyn Error>> {
+ use tokio::net::windows::named_pipe::ServerOptions;
+
+ let panic_location_file = test_panic(move || {
+ let rt = runtime_without_io();
+ rt.block_on(async {
+ let mut options = ServerOptions::new();
+ options.max_instances(255);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+// Runtime without `enable_io` so it has no IO driver set.
+fn runtime_without_io() -> Runtime {
+ Builder::new_current_thread().build().unwrap()
+}
diff --git a/tests/no_rt.rs b/tests/no_rt.rs
index 6845850..747fab6 100644
--- a/tests/no_rt.rs
+++ b/tests/no_rt.rs
@@ -1,3 +1,5 @@
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
+
use tokio::net::TcpStream;
use tokio::sync::oneshot;
use tokio::time::{timeout, Duration};
diff --git a/tests/process_raw_handle.rs b/tests/process_raw_handle.rs
index 727e66d..fbe22b0 100644
--- a/tests/process_raw_handle.rs
+++ b/tests/process_raw_handle.rs
@@ -3,7 +3,7 @@
#![cfg(windows)]
use tokio::process::Command;
-use winapi::um::processthreadsapi::GetProcessId;
+use windows_sys::Win32::System::Threading::GetProcessId;
#[tokio::test]
async fn obtain_raw_handle() {
diff --git a/tests/process_smoke.rs b/tests/process_smoke.rs
index fae5793..81d2020 100644
--- a/tests/process_smoke.rs
+++ b/tests/process_smoke.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi cannot run system commands
use tokio::process::Command;
use tokio_test::assert_ok;
diff --git a/tests/rt_basic.rs b/tests/rt_basic.rs
index 70056b1..6caf0a4 100644
--- a/tests/rt_basic.rs
+++ b/tests/rt_basic.rs
@@ -16,6 +16,15 @@
pub(crate) mod mpsc_stream;
}
+macro_rules! cfg_metrics {
+ ($($t:tt)*) => {
+ #[cfg(tokio_unstable)]
+ {
+ $( $t )*
+ }
+ }
+}
+
#[test]
fn spawned_task_does_not_progress_without_block_on() {
let (tx, mut rx) = oneshot::channel();
@@ -169,6 +178,105 @@
}
#[test]
+#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")]
+#[should_panic(expected = "boom")]
+fn wake_in_drop_after_panic() {
+ let (tx, rx) = oneshot::channel::<()>();
+
+ struct WakeOnDrop(Option<oneshot::Sender<()>>);
+
+ impl Drop for WakeOnDrop {
+ fn drop(&mut self) {
+ self.0.take().unwrap().send(()).unwrap();
+ }
+ }
+
+ let rt = rt();
+
+ rt.spawn(async move {
+ let _wake_on_drop = WakeOnDrop(Some(tx));
+ // wait forever
+ futures::future::pending::<()>().await;
+ });
+
+ let _join = rt.spawn(async move { rx.await });
+
+ rt.block_on(async {
+ tokio::task::yield_now().await;
+ panic!("boom");
+ });
+}
+
+#[test]
+fn spawn_two() {
+ let rt = rt();
+
+ let out = rt.block_on(async {
+ let (tx, rx) = oneshot::channel();
+
+ tokio::spawn(async move {
+ tokio::spawn(async move {
+ tx.send("ZOMG").unwrap();
+ });
+ });
+
+ assert_ok!(rx.await)
+ });
+
+ assert_eq!(out, "ZOMG");
+
+ cfg_metrics! {
+ let metrics = rt.metrics();
+ drop(rt);
+ assert_eq!(0, metrics.remote_schedule_count());
+
+ let mut local = 0;
+ for i in 0..metrics.num_workers() {
+ local += metrics.worker_local_schedule_count(i);
+ }
+
+ assert_eq!(2, local);
+ }
+}
+
+#[cfg_attr(tokio_wasi, ignore = "WASI: std::thread::spawn not supported")]
+#[test]
+fn spawn_remote() {
+ let rt = rt();
+
+ let out = rt.block_on(async {
+ let (tx, rx) = oneshot::channel();
+
+ let handle = tokio::spawn(async move {
+ std::thread::spawn(move || {
+ std::thread::sleep(Duration::from_millis(10));
+ tx.send("ZOMG").unwrap();
+ });
+
+ rx.await.unwrap()
+ });
+
+ handle.await.unwrap()
+ });
+
+ assert_eq!(out, "ZOMG");
+
+ cfg_metrics! {
+ let metrics = rt.metrics();
+ drop(rt);
+ assert_eq!(1, metrics.remote_schedule_count());
+
+ let mut local = 0;
+ for i in 0..metrics.num_workers() {
+ local += metrics.worker_local_schedule_count(i);
+ }
+
+ assert_eq!(1, local);
+ }
+}
+
+#[test]
+#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")]
#[should_panic(
expected = "A Tokio 1.x context was found, but timers are disabled. Call `enable_time` on the runtime builder to enable timers."
)]
@@ -183,6 +291,155 @@
});
}
+#[cfg(tokio_unstable)]
+mod unstable {
+ use tokio::runtime::{Builder, RngSeed, UnhandledPanic};
+
+ #[test]
+ #[should_panic(
+ expected = "a spawned task panicked and the runtime is configured to shut down on unhandled panic"
+ )]
+ fn shutdown_on_panic() {
+ let rt = Builder::new_current_thread()
+ .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ .build()
+ .unwrap();
+
+ rt.block_on(async {
+ tokio::spawn(async {
+ panic!("boom");
+ });
+
+ futures::future::pending::<()>().await;
+ })
+ }
+
+ #[test]
+ #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")]
+ fn spawns_do_nothing() {
+ use std::sync::Arc;
+
+ let rt = Builder::new_current_thread()
+ .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ .build()
+ .unwrap();
+
+ let rt1 = Arc::new(rt);
+ let rt2 = rt1.clone();
+
+ let _ = std::thread::spawn(move || {
+ rt2.block_on(async {
+ tokio::spawn(async {
+ panic!("boom");
+ });
+
+ futures::future::pending::<()>().await;
+ })
+ })
+ .join();
+
+ let task = rt1.spawn(async {});
+ let res = futures::executor::block_on(task);
+ assert!(res.is_err());
+ }
+
+ #[test]
+ #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")]
+ fn shutdown_all_concurrent_block_on() {
+ const N: usize = 2;
+ use std::sync::{mpsc, Arc};
+
+ let rt = Builder::new_current_thread()
+ .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ .build()
+ .unwrap();
+
+ let rt = Arc::new(rt);
+ let mut ths = vec![];
+ let (tx, rx) = mpsc::channel();
+
+ for _ in 0..N {
+ let rt = rt.clone();
+ let tx = tx.clone();
+ ths.push(std::thread::spawn(move || {
+ rt.block_on(async {
+ tx.send(()).unwrap();
+ futures::future::pending::<()>().await;
+ });
+ }));
+ }
+
+ for _ in 0..N {
+ rx.recv().unwrap();
+ }
+
+ rt.spawn(async {
+ panic!("boom");
+ });
+
+ for th in ths {
+ assert!(th.join().is_err());
+ }
+ }
+
+ #[test]
+ fn rng_seed() {
+ let seed = b"bytes used to generate seed";
+ let rt1 = tokio::runtime::Builder::new_current_thread()
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt1_values = rt1.block_on(async {
+ let rand_1 = tokio::macros::support::thread_rng_n(100);
+ let rand_2 = tokio::macros::support::thread_rng_n(100);
+
+ (rand_1, rand_2)
+ });
+
+ let rt2 = tokio::runtime::Builder::new_current_thread()
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt2_values = rt2.block_on(async {
+ let rand_1 = tokio::macros::support::thread_rng_n(100);
+ let rand_2 = tokio::macros::support::thread_rng_n(100);
+
+ (rand_1, rand_2)
+ });
+
+ assert_eq!(rt1_values, rt2_values);
+ }
+
+ #[test]
+ fn rng_seed_multi_enter() {
+ let seed = b"bytes used to generate seed";
+
+ fn two_rand_values() -> (u32, u32) {
+ let rand_1 = tokio::macros::support::thread_rng_n(100);
+ let rand_2 = tokio::macros::support::thread_rng_n(100);
+
+ (rand_1, rand_2)
+ }
+
+ let rt1 = tokio::runtime::Builder::new_current_thread()
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt1_values_1 = rt1.block_on(async { two_rand_values() });
+ let rt1_values_2 = rt1.block_on(async { two_rand_values() });
+
+ let rt2 = tokio::runtime::Builder::new_current_thread()
+ .rng_seed(RngSeed::from_bytes(seed))
+ .build()
+ .unwrap();
+ let rt2_values_1 = rt2.block_on(async { two_rand_values() });
+ let rt2_values_2 = rt2.block_on(async { two_rand_values() });
+
+ assert_eq!(rt1_values_1, rt2_values_1);
+ assert_eq!(rt1_values_2, rt2_values_2);
+ }
+}
+
fn rt() -> Runtime {
tokio::runtime::Builder::new_current_thread()
.enable_all()
diff --git a/tests/rt_common.rs b/tests/rt_common.rs
index e5fc7a9..3892998 100644
--- a/tests/rt_common.rs
+++ b/tests/rt_common.rs
@@ -2,13 +2,16 @@
#![warn(rust_2018_idioms)]
#![cfg(feature = "full")]
-// Tests to run on both current-thread & thread-pool runtime variants.
+// Tests to run on both current-thread & multi-thread runtime variants.
macro_rules! rt_test {
($($t:tt)*) => {
mod current_thread_scheduler {
$($t)*
+ #[cfg(not(target_os="wasi"))]
+ const NUM_WORKERS: usize = 1;
+
fn rt() -> Arc<Runtime> {
tokio::runtime::Builder::new_current_thread()
.enable_all()
@@ -18,9 +21,12 @@
}
}
+ #[cfg(not(tokio_wasi))] // Wasi doesn't support threads
mod threaded_scheduler_4_threads {
$($t)*
+ const NUM_WORKERS: usize = 4;
+
fn rt() -> Arc<Runtime> {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
@@ -31,9 +37,12 @@
}
}
+ #[cfg(not(tokio_wasi))] // Wasi doesn't support threads
mod threaded_scheduler_1_thread {
$($t)*
+ const NUM_WORKERS: usize = 1;
+
fn rt() -> Arc<Runtime> {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
@@ -55,18 +64,30 @@
}
rt_test! {
- use tokio::net::{TcpListener, TcpStream, UdpSocket};
+ #[cfg(not(target_os="wasi"))]
+ use tokio::net::{TcpListener, TcpStream};
+ #[cfg(not(target_os="wasi"))]
use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
use tokio::runtime::Runtime;
use tokio::sync::oneshot;
use tokio::{task, time};
- use tokio_test::{assert_err, assert_ok};
+
+ #[cfg(not(target_os="wasi"))]
+ use tokio_test::assert_err;
+ use tokio_test::assert_ok;
use futures::future::poll_fn;
use std::future::Future;
use std::pin::Pin;
- use std::sync::{mpsc, Arc};
+
+ #[cfg(not(target_os="wasi"))]
+ use std::sync::mpsc;
+
+ use std::sync::Arc;
use std::task::{Context, Poll};
+
+ #[cfg(not(target_os="wasi"))]
use std::thread;
use std::time::{Duration, Instant};
@@ -83,6 +104,7 @@
}
+ #[cfg(not(target_os="wasi"))]
#[test]
fn block_on_async() {
let rt = rt();
@@ -164,6 +186,7 @@
assert_eq!(out, "ZOMG");
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn spawn_many_from_block_on() {
use tokio::sync::mpsc;
@@ -214,6 +237,7 @@
}
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn spawn_many_from_task() {
use tokio::sync::mpsc;
@@ -226,14 +250,6 @@
tokio::spawn(async move {
let (done_tx, mut done_rx) = mpsc::unbounded_channel();
- /*
- for _ in 0..100 {
- tokio::spawn(async move { });
- }
-
- tokio::task::yield_now().await;
- */
-
let mut txs = (0..ITER)
.map(|i| {
let (tx, rx) = oneshot::channel();
@@ -275,6 +291,31 @@
}
#[test]
+ fn spawn_one_from_block_on_called_on_handle() {
+ let rt = rt();
+ let (tx, rx) = oneshot::channel();
+
+ #[allow(clippy::async_yields_async)]
+ let handle = rt.handle().block_on(async {
+ tokio::spawn(async move {
+ tx.send("ZOMG").unwrap();
+ "DONE"
+ })
+ });
+
+ let out = rt.block_on(async {
+ let msg = assert_ok!(rx.await);
+
+ let out = assert_ok!(handle.await);
+ assert_eq!(out, "DONE");
+
+ msg
+ });
+
+ assert_eq!(out, "ZOMG");
+ }
+
+ #[test]
fn spawn_await_chain() {
let rt = rt();
@@ -329,6 +370,7 @@
assert_eq!(out, "ZOMG");
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn complete_block_on_under_load() {
let rt = rt();
@@ -352,6 +394,7 @@
});
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn complete_task_under_load() {
let rt = rt();
@@ -381,6 +424,7 @@
});
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn spawn_from_other_thread_idle() {
let rt = rt();
@@ -401,6 +445,7 @@
});
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn spawn_from_other_thread_under_load() {
let rt = rt();
@@ -461,6 +506,7 @@
assert!(now.elapsed() >= dur);
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support bind
#[test]
fn block_on_socket() {
let rt = rt();
@@ -481,6 +527,7 @@
});
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn spawn_from_blocking() {
let rt = rt();
@@ -496,6 +543,7 @@
assert_eq!(out, "hello")
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn spawn_blocking_from_blocking() {
let rt = rt();
@@ -511,6 +559,7 @@
assert_eq!(out, "hello")
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn sleep_from_blocking() {
let rt = rt();
@@ -531,6 +580,7 @@
});
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support bind
#[test]
fn socket_from_blocking() {
let rt = rt();
@@ -554,6 +604,7 @@
});
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn always_active_parker() {
// This test it to show that we will always have
@@ -600,6 +651,7 @@
// concern. There also isn't a great/obvious solution to take. For now, the
// test is disabled.
#[cfg(not(windows))]
+ #[cfg(not(target_os="wasi"))] // Wasi does not support bind or threads
fn io_driver_called_when_under_load() {
let rt = rt();
@@ -607,7 +659,12 @@
for _ in 0..100 {
rt.spawn(async {
loop {
- tokio::task::yield_now().await;
+ // Don't use Tokio's `yield_now()` to avoid special defer
+ // logic.
+ futures::future::poll_fn::<(), _>(|cx| {
+ cx.waker().wake_by_ref();
+ std::task::Poll::Pending
+ }).await;
}
});
}
@@ -635,6 +692,72 @@
});
}
+ /// Tests that yielded tasks are not scheduled until **after** resource
+ /// drivers are polled.
+ ///
+ /// Note: we may have to delete this test as it is not necessarily reliable.
+ /// The OS does not guarantee when I/O events are delivered, so there may be
+ /// more yields than anticipated.
+ #[test]
+ #[cfg(not(target_os="wasi"))]
+ fn yield_defers_until_park() {
+ use std::sync::atomic::{AtomicBool, Ordering::SeqCst};
+ use std::sync::Barrier;
+
+ let rt = rt();
+
+ let flag = Arc::new(AtomicBool::new(false));
+ let barrier = Arc::new(Barrier::new(NUM_WORKERS));
+
+ rt.block_on(async {
+ // Make sure other workers cannot steal tasks
+ #[allow(clippy::reversed_empty_ranges)]
+ for _ in 0..(NUM_WORKERS-1) {
+ let flag = flag.clone();
+ let barrier = barrier.clone();
+
+ tokio::spawn(async move {
+ barrier.wait();
+
+ while !flag.load(SeqCst) {
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ });
+ }
+
+ barrier.wait();
+
+ tokio::spawn(async move {
+ // Create a TCP litener
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+
+ tokio::join!(
+ async {
+ // Done blocking intentionally
+ let _socket = std::net::TcpStream::connect(addr).unwrap();
+
+ // Yield until connected
+ let mut cnt = 0;
+ while !flag.load(SeqCst){
+ tokio::task::yield_now().await;
+ cnt += 1;
+
+ if cnt >= 10 {
+ panic!("yielded too many times; TODO: delete this test?");
+ }
+ }
+ },
+ async {
+ let _ = listener.accept().await.unwrap();
+ flag.store(true, SeqCst);
+ }
+ );
+ }).await.unwrap();
+ });
+ }
+
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn client_server_block_on() {
let rt = rt();
@@ -646,6 +769,7 @@
assert_err!(rx.try_recv());
}
+ #[cfg_attr(tokio_wasi, ignore = "Wasi does not support threads or panic recovery")]
#[test]
#[cfg(not(target_os = "android"))]
fn panic_in_task() {
@@ -675,11 +799,13 @@
#[test]
#[should_panic]
+ #[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")]
fn panic_in_block_on() {
let rt = rt();
rt.block_on(async { panic!() });
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
async fn yield_once() {
let mut yielded = false;
poll_fn(|cx| {
@@ -749,7 +875,11 @@
#[test]
fn wake_while_rt_is_dropping() {
- use tokio::task;
+ use tokio::sync::Barrier;
+ use core::sync::atomic::{AtomicBool, Ordering};
+
+ let drop_triggered = Arc::new(AtomicBool::new(false));
+ let set_drop_triggered = drop_triggered.clone();
struct OnDrop<F: FnMut()>(F);
@@ -763,17 +893,21 @@
let (tx2, rx2) = oneshot::channel();
let (tx3, rx3) = oneshot::channel();
- let rt = rt();
+ let barrier = Arc::new(Barrier::new(4));
+ let barrier1 = barrier.clone();
+ let barrier2 = barrier.clone();
+ let barrier3 = barrier.clone();
- let h1 = rt.clone();
+ let rt = rt();
rt.spawn(async move {
// Ensure a waker gets stored in oneshot 1.
- let _ = rx1.await;
+ let _ = tokio::join!(rx1, barrier1.wait());
tx3.send(()).unwrap();
});
rt.spawn(async move {
+ let h1 = tokio::runtime::Handle::current();
// When this task is dropped, we'll be "closing remotes".
// We spawn a new task that owns the `tx1`, to move its Drop
// out of here.
@@ -786,36 +920,40 @@
h1.spawn(async move {
tx1.send(()).unwrap();
});
+ // Just a sanity check that this entire thing actually happened
+ set_drop_triggered.store(true, Ordering::Relaxed);
});
- let _ = rx2.await;
+ let _ = tokio::join!(rx2, barrier2.wait());
});
rt.spawn(async move {
- let _ = rx3.await;
+ let _ = tokio::join!(rx3, barrier3.wait());
// We'll never get here, but once task 3 drops, this will
// force task 2 to re-schedule since it's waiting on oneshot 2.
tx2.send(()).unwrap();
});
- // Tick the loop
- rt.block_on(async {
- task::yield_now().await;
- });
+ // Wait until every oneshot channel has been polled.
+ rt.block_on(barrier.wait());
// Drop the rt
drop(rt);
+
+ // Make sure that the spawn actually happened
+ assert!(drop_triggered.load(Ordering::Relaxed));
}
+ #[cfg(not(target_os="wasi"))] // Wasi doesn't support UDP or bind()
#[test]
fn io_notify_while_shutting_down() {
- use std::net::Ipv6Addr;
+ use tokio::net::UdpSocket;
use std::sync::Arc;
for _ in 1..10 {
let runtime = rt();
runtime.block_on(async {
- let socket = UdpSocket::bind((Ipv6Addr::LOCALHOST, 0)).await.unwrap();
+ let socket = UdpSocket::bind("127.0.0.1:0").await.unwrap();
let addr = socket.local_addr().unwrap();
let send_half = Arc::new(socket);
let recv_half = send_half.clone();
@@ -841,6 +979,7 @@
}
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn shutdown_timeout() {
let (tx, rx) = oneshot::channel();
@@ -858,6 +997,7 @@
Arc::try_unwrap(runtime).unwrap().shutdown_timeout(Duration::from_millis(100));
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support threads
#[test]
fn shutdown_timeout_0() {
let runtime = rt();
@@ -889,6 +1029,7 @@
// See https://github.com/rust-lang/rust/issues/74875
#[test]
#[cfg(not(windows))]
+ #[cfg_attr(tokio_wasi, ignore = "Wasi does not support threads")]
fn runtime_in_thread_local() {
use std::cell::RefCell;
use std::thread;
@@ -908,6 +1049,7 @@
}).join().unwrap();
}
+ #[cfg(not(target_os="wasi"))] // Wasi does not support bind
async fn client_server(tx: mpsc::Sender<()>) {
let server = assert_ok!(TcpListener::bind("127.0.0.1:0").await);
@@ -932,6 +1074,7 @@
tx.send(()).unwrap();
}
+ #[cfg(not(tokio_wasi))] // Wasi does not support bind
#[test]
fn local_set_block_on_socket() {
let rt = rt();
@@ -953,6 +1096,7 @@
});
}
+ #[cfg(not(tokio_wasi))] // Wasi does not support bind
#[test]
fn local_set_client_server_block_on() {
let rt = rt();
@@ -966,6 +1110,7 @@
assert_err!(rx.try_recv());
}
+ #[cfg(not(tokio_wasi))] // Wasi does not support bind
async fn client_server_local(tx: mpsc::Sender<()>) {
let server = assert_ok!(TcpListener::bind("127.0.0.1:0").await);
@@ -993,22 +1138,22 @@
#[test]
fn coop() {
use std::task::Poll::Ready;
+ use tokio::sync::mpsc;
let rt = rt();
rt.block_on(async {
- // Create a bunch of tasks
- let mut tasks = (0..1_000).map(|_| {
- tokio::spawn(async { })
- }).collect::<Vec<_>>();
+ let (send, mut recv) = mpsc::unbounded_channel();
- // Hope that all the tasks complete...
- time::sleep(Duration::from_millis(100)).await;
+ // Send a bunch of messages.
+ for _ in 0..1_000 {
+ send.send(()).unwrap();
+ }
poll_fn(|cx| {
- // At least one task should not be ready
- for task in &mut tasks {
- if Pin::new(task).poll(cx).is_pending() {
+ // At least one response should return pending.
+ for _ in 0..1_000 {
+ if recv.poll_recv(cx).is_pending() {
return Ready(());
}
}
@@ -1021,22 +1166,22 @@
#[test]
fn coop_unconstrained() {
use std::task::Poll::Ready;
+ use tokio::sync::mpsc;
let rt = rt();
rt.block_on(async {
- // Create a bunch of tasks
- let mut tasks = (0..1_000).map(|_| {
- tokio::spawn(async { })
- }).collect::<Vec<_>>();
+ let (send, mut recv) = mpsc::unbounded_channel();
- // Hope that all the tasks complete...
- time::sleep(Duration::from_millis(100)).await;
+ // Send a bunch of messages.
+ for _ in 0..1_000 {
+ send.send(()).unwrap();
+ }
tokio::task::unconstrained(poll_fn(|cx| {
- // All the tasks should be ready
- for task in &mut tasks {
- assert!(Pin::new(task).poll(cx).is_ready());
+ // All the responses should be ready.
+ for _ in 0..1_000 {
+ assert_eq!(recv.poll_recv(cx), Poll::Ready(Some(())));
}
Ready(())
@@ -1044,6 +1189,31 @@
});
}
+ #[cfg(tokio_unstable)]
+ #[test]
+ fn coop_consume_budget() {
+ let rt = rt();
+
+ rt.block_on(async {
+ poll_fn(|cx| {
+ let counter = Arc::new(std::sync::Mutex::new(0));
+ let counter_clone = Arc::clone(&counter);
+ let mut worker = Box::pin(async move {
+ // Consume the budget until a yield happens
+ for _ in 0..1000 {
+ *counter.lock().unwrap() += 1;
+ task::consume_budget().await
+ }
+ });
+ // Assert that the worker was yielded and it didn't manage
+ // to finish the whole work (assuming the total budget of 128)
+ assert!(Pin::new(&mut worker).poll(cx).is_pending());
+ assert!(*counter_clone.lock().unwrap() < 1000);
+ std::task::Poll::Ready(())
+ }).await;
+ });
+ }
+
// Tests that the "next task" scheduler optimization is not able to starve
// other tasks.
#[test]
@@ -1061,7 +1231,7 @@
let (spawned_tx, mut spawned_rx) = mpsc::unbounded_channel();
let mut tasks = vec![];
- // Spawn a bunch of tasks that ping ping between each other to
+ // Spawn a bunch of tasks that ping-pong between each other to
// saturate the runtime.
for _ in 0..NUM {
let (tx1, mut rx1) = mpsc::unbounded_channel();
diff --git a/tests/rt_handle_block_on.rs b/tests/rt_handle_block_on.rs
index 17878c8..5ec783e 100644
--- a/tests/rt_handle_block_on.rs
+++ b/tests/rt_handle_block_on.rs
@@ -10,9 +10,10 @@
use std::time::Duration;
use tokio::runtime::{Handle, Runtime};
use tokio::sync::mpsc;
-use tokio::task::spawn_blocking;
-use tokio::{fs, net, time};
+#[cfg(not(tokio_wasi))]
+use tokio::{net, time};
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
macro_rules! multi_threaded_rt_test {
($($t:tt)*) => {
mod threaded_scheduler_4_threads_only {
@@ -45,6 +46,7 @@
}
}
+#[cfg(not(tokio_wasi))]
macro_rules! rt_test {
($($t:tt)*) => {
mod current_thread_scheduler {
@@ -124,7 +126,9 @@
})
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support file operations or bind
rt_test! {
+ use tokio::fs;
// ==== spawn blocking futures ======
#[test]
@@ -135,7 +139,7 @@
let contents = Handle::current()
.block_on(fs::read_to_string("Cargo.toml"))
.unwrap();
- assert!(contents.contains("Cargo.toml"));
+ assert!(contents.contains("https://tokio.rs"));
}
#[test]
@@ -156,6 +160,7 @@
#[test]
fn basic_spawn_blocking() {
+ use tokio::task::spawn_blocking;
let rt = rt();
let _enter = rt.enter();
@@ -171,6 +176,7 @@
#[test]
fn spawn_blocking_after_shutdown_fails() {
+ use tokio::task::spawn_blocking;
let rt = rt();
let _enter = rt.enter();
rt.shutdown_timeout(Duration::from_secs(1000));
@@ -187,6 +193,7 @@
#[test]
fn spawn_blocking_started_before_shutdown_continues() {
+ use tokio::task::spawn_blocking;
let rt = rt();
let _enter = rt.enter();
@@ -412,6 +419,7 @@
}
}
+#[cfg(not(tokio_wasi))]
multi_threaded_rt_test! {
#[cfg(unix)]
#[test]
@@ -474,6 +482,7 @@
// ==== utils ======
/// Create a new multi threaded runtime
+#[cfg(not(tokio_wasi))]
fn new_multi_thread(n: usize) -> Runtime {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(n)
@@ -496,37 +505,30 @@
F: Fn(),
{
{
- println!("current thread runtime");
-
let rt = new_current_thread();
let _enter = rt.enter();
f();
- println!("current thread runtime after shutdown");
rt.shutdown_timeout(Duration::from_secs(1000));
f();
}
+ #[cfg(not(tokio_wasi))]
{
- println!("multi thread (1 thread) runtime");
-
let rt = new_multi_thread(1);
let _enter = rt.enter();
f();
- println!("multi thread runtime after shutdown");
rt.shutdown_timeout(Duration::from_secs(1000));
f();
}
+ #[cfg(not(tokio_wasi))]
{
- println!("multi thread (4 threads) runtime");
-
let rt = new_multi_thread(4);
let _enter = rt.enter();
f();
- println!("multi thread runtime after shutdown");
rt.shutdown_timeout(Duration::from_secs(1000));
f();
}
diff --git a/tests/rt_metrics.rs b/tests/rt_metrics.rs
new file mode 100644
index 0000000..fdb2fb5
--- /dev/null
+++ b/tests/rt_metrics.rs
@@ -0,0 +1,481 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", tokio_unstable, not(tokio_wasi)))]
+
+use std::sync::{Arc, Mutex};
+
+use tokio::runtime::Runtime;
+use tokio::time::{self, Duration};
+
+#[test]
+fn num_workers() {
+ let rt = current_thread();
+ assert_eq!(1, rt.metrics().num_workers());
+
+ let rt = threaded();
+ assert_eq!(2, rt.metrics().num_workers());
+}
+
+#[test]
+fn num_blocking_threads() {
+ let rt = current_thread();
+ assert_eq!(0, rt.metrics().num_blocking_threads());
+ let _ = rt.block_on(rt.spawn_blocking(move || {}));
+ assert_eq!(1, rt.metrics().num_blocking_threads());
+}
+
+#[test]
+fn num_idle_blocking_threads() {
+ let rt = current_thread();
+ assert_eq!(0, rt.metrics().num_idle_blocking_threads());
+ let _ = rt.block_on(rt.spawn_blocking(move || {}));
+ rt.block_on(async {
+ time::sleep(Duration::from_millis(5)).await;
+ });
+
+ // We need to wait until the blocking thread has become idle. Usually 5ms is
+ // enough for this to happen, but not always. When it isn't enough, sleep
+ // for another second. We don't always wait for a whole second since we want
+ // the test suite to finish quickly.
+ //
+ // Note that the timeout for idle threads to be killed is 10 seconds.
+ if 0 == rt.metrics().num_idle_blocking_threads() {
+ rt.block_on(async {
+ time::sleep(Duration::from_secs(1)).await;
+ });
+ }
+
+ assert_eq!(1, rt.metrics().num_idle_blocking_threads());
+}
+
+#[test]
+fn blocking_queue_depth() {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .max_blocking_threads(1)
+ .build()
+ .unwrap();
+
+ assert_eq!(0, rt.metrics().blocking_queue_depth());
+
+ let ready = Arc::new(Mutex::new(()));
+ let guard = ready.lock().unwrap();
+
+ let ready_cloned = ready.clone();
+ let wait_until_ready = move || {
+ let _unused = ready_cloned.lock().unwrap();
+ };
+
+ let h1 = rt.spawn_blocking(wait_until_ready.clone());
+ let h2 = rt.spawn_blocking(wait_until_ready);
+ assert!(rt.metrics().blocking_queue_depth() > 0);
+
+ drop(guard);
+
+ let _ = rt.block_on(h1);
+ let _ = rt.block_on(h2);
+
+ assert_eq!(0, rt.metrics().blocking_queue_depth());
+}
+
+#[test]
+fn remote_schedule_count() {
+ use std::thread;
+
+ let rt = current_thread();
+ let handle = rt.handle().clone();
+ let task = thread::spawn(move || {
+ handle.spawn(async {
+ // DO nothing
+ })
+ })
+ .join()
+ .unwrap();
+
+ rt.block_on(task).unwrap();
+
+ assert_eq!(1, rt.metrics().remote_schedule_count());
+
+ let rt = threaded();
+ let handle = rt.handle().clone();
+ let task = thread::spawn(move || {
+ handle.spawn(async {
+ // DO nothing
+ })
+ })
+ .join()
+ .unwrap();
+
+ rt.block_on(task).unwrap();
+
+ assert_eq!(1, rt.metrics().remote_schedule_count());
+}
+
+#[test]
+fn worker_park_count() {
+ let rt = current_thread();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ time::sleep(Duration::from_millis(1)).await;
+ });
+ drop(rt);
+ assert!(1 <= metrics.worker_park_count(0));
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ time::sleep(Duration::from_millis(1)).await;
+ });
+ drop(rt);
+ assert!(1 <= metrics.worker_park_count(0));
+ assert!(1 <= metrics.worker_park_count(1));
+}
+
+#[test]
+fn worker_noop_count() {
+ // There isn't really a great way to generate no-op parks as they happen as
+ // false-positive events under concurrency.
+
+ let rt = current_thread();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ time::sleep(Duration::from_millis(1)).await;
+ });
+ drop(rt);
+ assert!(0 < metrics.worker_noop_count(0));
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ time::sleep(Duration::from_millis(1)).await;
+ });
+ drop(rt);
+ assert!(0 < metrics.worker_noop_count(0));
+ assert!(0 < metrics.worker_noop_count(1));
+}
+
+#[test]
+fn worker_steal_count() {
+ // This metric only applies to the multi-threaded runtime.
+ //
+ // We use a blocking channel to backup one worker thread.
+ use std::sync::mpsc::channel;
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+
+ rt.block_on(async {
+ let (tx, rx) = channel();
+
+ // Move to the runtime.
+ tokio::spawn(async move {
+ // Spawn the task that sends to the channel
+ tokio::spawn(async move {
+ tx.send(()).unwrap();
+ });
+
+ // Spawn a task that bumps the previous task out of the "next
+ // scheduled" slot.
+ tokio::spawn(async {});
+
+ // Blocking receive on the channel.
+ rx.recv().unwrap();
+ })
+ .await
+ .unwrap();
+ });
+
+ drop(rt);
+
+ let n: u64 = (0..metrics.num_workers())
+ .map(|i| metrics.worker_steal_count(i))
+ .sum();
+
+ assert_eq!(1, n);
+}
+
+#[test]
+fn worker_poll_count() {
+ const N: u64 = 5;
+
+ let rt = current_thread();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ for _ in 0..N {
+ tokio::spawn(async {}).await.unwrap();
+ }
+ });
+ drop(rt);
+ assert_eq!(N, metrics.worker_poll_count(0));
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ for _ in 0..N {
+ tokio::spawn(async {}).await.unwrap();
+ }
+ });
+ drop(rt);
+ // Account for the `block_on` task
+ let n = (0..metrics.num_workers())
+ .map(|i| metrics.worker_poll_count(i))
+ .sum();
+
+ assert_eq!(N, n);
+}
+
+#[test]
+fn worker_total_busy_duration() {
+ const N: usize = 5;
+
+ let zero = Duration::from_millis(0);
+
+ let rt = current_thread();
+ let metrics = rt.metrics();
+
+ rt.block_on(async {
+ for _ in 0..N {
+ tokio::spawn(async {
+ tokio::task::yield_now().await;
+ })
+ .await
+ .unwrap();
+ }
+ });
+
+ drop(rt);
+
+ assert!(zero < metrics.worker_total_busy_duration(0));
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+
+ rt.block_on(async {
+ for _ in 0..N {
+ tokio::spawn(async {
+ tokio::task::yield_now().await;
+ })
+ .await
+ .unwrap();
+ }
+ });
+
+ drop(rt);
+
+ for i in 0..metrics.num_workers() {
+ assert!(zero < metrics.worker_total_busy_duration(i));
+ }
+}
+
+#[test]
+fn worker_local_schedule_count() {
+ let rt = current_thread();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ tokio::spawn(async {}).await.unwrap();
+ });
+ drop(rt);
+
+ assert_eq!(1, metrics.worker_local_schedule_count(0));
+ assert_eq!(0, metrics.remote_schedule_count());
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ // Move to the runtime
+ tokio::spawn(async {
+ tokio::spawn(async {}).await.unwrap();
+ })
+ .await
+ .unwrap();
+ });
+ drop(rt);
+
+ let n: u64 = (0..metrics.num_workers())
+ .map(|i| metrics.worker_local_schedule_count(i))
+ .sum();
+
+ assert_eq!(2, n);
+ assert_eq!(1, metrics.remote_schedule_count());
+}
+
+#[test]
+fn worker_overflow_count() {
+ // Only applies to the threaded worker
+ let rt = threaded();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ // Move to the runtime
+ tokio::spawn(async {
+ let (tx1, rx1) = std::sync::mpsc::channel();
+ let (tx2, rx2) = std::sync::mpsc::channel();
+
+ // First, we need to block the other worker until all tasks have
+ // been spawned.
+ tokio::spawn(async move {
+ tx1.send(()).unwrap();
+ rx2.recv().unwrap();
+ });
+
+ // Bump the next-run spawn
+ tokio::spawn(async {});
+
+ rx1.recv().unwrap();
+
+ // Spawn many tasks
+ for _ in 0..300 {
+ tokio::spawn(async {});
+ }
+
+ tx2.send(()).unwrap();
+ })
+ .await
+ .unwrap();
+ });
+ drop(rt);
+
+ let n: u64 = (0..metrics.num_workers())
+ .map(|i| metrics.worker_overflow_count(i))
+ .sum();
+
+ assert_eq!(1, n);
+}
+
+#[test]
+fn injection_queue_depth() {
+ use std::thread;
+
+ let rt = current_thread();
+ let handle = rt.handle().clone();
+ let metrics = rt.metrics();
+
+ thread::spawn(move || {
+ handle.spawn(async {});
+ })
+ .join()
+ .unwrap();
+
+ assert_eq!(1, metrics.injection_queue_depth());
+
+ let rt = threaded();
+ let handle = rt.handle().clone();
+ let metrics = rt.metrics();
+
+ // First we need to block the runtime workers
+ let (tx1, rx1) = std::sync::mpsc::channel();
+ let (tx2, rx2) = std::sync::mpsc::channel();
+
+ rt.spawn(async move { rx1.recv().unwrap() });
+ rt.spawn(async move { rx2.recv().unwrap() });
+
+ thread::spawn(move || {
+ handle.spawn(async {});
+ })
+ .join()
+ .unwrap();
+
+ let n = metrics.injection_queue_depth();
+ assert!(1 <= n, "{}", n);
+ assert!(3 >= n, "{}", n);
+
+ tx1.send(()).unwrap();
+ tx2.send(()).unwrap();
+}
+
+#[test]
+fn worker_local_queue_depth() {
+ const N: usize = 100;
+
+ let rt = current_thread();
+ let metrics = rt.metrics();
+ rt.block_on(async {
+ for _ in 0..N {
+ tokio::spawn(async {});
+ }
+
+ assert_eq!(N, metrics.worker_local_queue_depth(0));
+ });
+
+ let rt = threaded();
+ let metrics = rt.metrics();
+ rt.block_on(async move {
+ // Move to the runtime
+ tokio::spawn(async move {
+ let (tx1, rx1) = std::sync::mpsc::channel();
+ let (tx2, rx2) = std::sync::mpsc::channel();
+
+ // First, we need to block the other worker until all tasks have
+ // been spawned.
+ tokio::spawn(async move {
+ tx1.send(()).unwrap();
+ rx2.recv().unwrap();
+ });
+
+ // Bump the next-run spawn
+ tokio::spawn(async {});
+
+ rx1.recv().unwrap();
+
+ // Spawn some tasks
+ for _ in 0..100 {
+ tokio::spawn(async {});
+ }
+
+ let n: usize = (0..metrics.num_workers())
+ .map(|i| metrics.worker_local_queue_depth(i))
+ .sum();
+
+ assert_eq!(n, N);
+
+ tx2.send(()).unwrap();
+ })
+ .await
+ .unwrap();
+ });
+}
+
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+#[test]
+fn io_driver_fd_count() {
+ let rt = current_thread();
+ let metrics = rt.metrics();
+
+ assert_eq!(metrics.io_driver_fd_registered_count(), 0);
+
+ let stream = tokio::net::TcpStream::connect("google.com:80");
+ let stream = rt.block_on(async move { stream.await.unwrap() });
+
+ assert_eq!(metrics.io_driver_fd_registered_count(), 1);
+ assert_eq!(metrics.io_driver_fd_deregistered_count(), 0);
+
+ drop(stream);
+
+ assert_eq!(metrics.io_driver_fd_deregistered_count(), 1);
+ assert_eq!(metrics.io_driver_fd_registered_count(), 1);
+}
+
+#[cfg(any(target_os = "linux", target_os = "macos"))]
+#[test]
+fn io_driver_ready_count() {
+ let rt = current_thread();
+ let metrics = rt.metrics();
+
+ let stream = tokio::net::TcpStream::connect("google.com:80");
+ let _stream = rt.block_on(async move { stream.await.unwrap() });
+
+ assert_eq!(metrics.io_driver_ready_count(), 1);
+}
+
+fn current_thread() -> Runtime {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+}
+
+fn threaded() -> Runtime {
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(2)
+ .enable_all()
+ .build()
+ .unwrap()
+}
diff --git a/tests/rt_panic.rs b/tests/rt_panic.rs
new file mode 100644
index 0000000..f9a684f
--- /dev/null
+++ b/tests/rt_panic.rs
@@ -0,0 +1,77 @@
+#![warn(rust_2018_idioms)]
+#![cfg(feature = "full")]
+#![cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery
+
+use futures::future;
+use std::error::Error;
+use tokio::runtime::{Builder, Handle, Runtime};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+#[test]
+fn current_handle_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = Handle::current();
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn into_panic_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(move || {
+ let rt = current_thread();
+ rt.block_on(async {
+ let handle = tokio::spawn(future::pending::<()>());
+
+ handle.abort();
+
+ let err = handle.await.unwrap_err();
+ assert!(!&err.is_panic());
+
+ let _ = err.into_panic();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn builder_worker_threads_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = Builder::new_multi_thread().worker_threads(0).build();
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn builder_max_blocking_threads_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = Builder::new_multi_thread().max_blocking_threads(0).build();
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+fn current_thread() -> Runtime {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+}
diff --git a/tests/rt_threaded.rs b/tests/rt_threaded.rs
index 5f047a7..c598418 100644
--- a/tests/rt_threaded.rs
+++ b/tests/rt_threaded.rs
@@ -1,9 +1,9 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))]
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
-use tokio::runtime::{self, Runtime};
+use tokio::runtime;
use tokio::sync::oneshot;
use tokio_test::{assert_err, assert_ok};
@@ -15,6 +15,15 @@
use std::sync::{mpsc, Arc, Mutex};
use std::task::{Context, Poll, Waker};
+macro_rules! cfg_metrics {
+ ($($t:tt)*) => {
+ #[cfg(tokio_unstable)]
+ {
+ $( $t )*
+ }
+ }
+}
+
#[test]
fn single_thread() {
// No panic when starting a runtime w/ a single thread
@@ -56,6 +65,38 @@
}
#[test]
+fn spawn_two() {
+ let rt = rt();
+
+ let out = rt.block_on(async {
+ let (tx, rx) = oneshot::channel();
+
+ tokio::spawn(async move {
+ tokio::spawn(async move {
+ tx.send("ZOMG").unwrap();
+ });
+ });
+
+ assert_ok!(rx.await)
+ });
+
+ assert_eq!(out, "ZOMG");
+
+ cfg_metrics! {
+ let metrics = rt.metrics();
+ drop(rt);
+ assert_eq!(1, metrics.remote_schedule_count());
+
+ let mut local = 0;
+ for i in 0..metrics.num_workers() {
+ local += metrics.worker_local_schedule_count(i);
+ }
+
+ assert_eq!(1, local);
+ }
+}
+
+#[test]
fn many_multishot_futures() {
const CHAIN: usize = 200;
const CYCLES: usize = 5;
@@ -374,6 +415,32 @@
});
}
+#[test]
+fn yield_after_block_in_place() {
+ let rt = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
+ .build()
+ .unwrap();
+
+ rt.block_on(async {
+ tokio::spawn(async move {
+ // Block in place then enter a new runtime
+ tokio::task::block_in_place(|| {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap();
+
+ rt.block_on(async {});
+ });
+
+ // Yield, then complete
+ tokio::task::yield_now().await;
+ })
+ .await
+ .unwrap()
+ });
+}
+
// Testing this does not panic
#[test]
fn max_blocking_threads() {
@@ -439,9 +506,7 @@
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
let me = Pin::into_inner(self);
let mut lock = me.shared.lock().unwrap();
- println!("poll {}", me.put_waker);
if me.put_waker {
- println!("putting");
lock.waker = Some(cx.waker().clone());
}
Poll::Pending
@@ -450,13 +515,11 @@
impl Drop for MyFuture {
fn drop(&mut self) {
- println!("drop {} start", self.put_waker);
let mut lock = self.shared.lock().unwrap();
if !self.put_waker {
lock.waker.take().unwrap().wake();
}
drop(lock);
- println!("drop {} stop", self.put_waker);
}
}
@@ -498,6 +561,30 @@
tokio::task::block_in_place(|| {});
}
-fn rt() -> Runtime {
- Runtime::new().unwrap()
+fn rt() -> runtime::Runtime {
+ runtime::Runtime::new().unwrap()
+}
+
+#[cfg(tokio_unstable)]
+mod unstable {
+ use super::*;
+
+ #[test]
+ fn test_disable_lifo_slot() {
+ let rt = runtime::Builder::new_multi_thread()
+ .disable_lifo_slot()
+ .worker_threads(2)
+ .build()
+ .unwrap();
+
+ rt.block_on(async {
+ tokio::spawn(async {
+ // Spawn another task and block the thread until completion. If the LIFO slot
+ // is used then the test doesn't complete.
+ futures::executor::block_on(tokio::spawn(async {})).unwrap();
+ })
+ .await
+ .unwrap();
+ })
+ }
}
diff --git a/tests/signal_no_rt.rs b/tests/signal_no_rt.rs
index b0f32b2..ffcc665 100644
--- a/tests/signal_no_rt.rs
+++ b/tests/signal_no_rt.rs
@@ -4,6 +4,7 @@
use tokio::signal::unix::{signal, SignalKind};
+#[cfg_attr(tokio_wasi, ignore = "Wasi does not support panic recovery")]
#[test]
#[should_panic]
fn no_runtime_panics_creating_signals() {
diff --git a/tests/signal_panic.rs b/tests/signal_panic.rs
new file mode 100644
index 0000000..ce1ec3e
--- /dev/null
+++ b/tests/signal_panic.rs
@@ -0,0 +1,29 @@
+#![warn(rust_2018_idioms)]
+#![cfg(feature = "full")]
+#![cfg(unix)]
+
+use std::error::Error;
+use tokio::runtime::Builder;
+use tokio::signal::unix::{signal, SignalKind};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+#[test]
+fn signal_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = Builder::new_current_thread().build().unwrap();
+
+ rt.block_on(async {
+ let kind = SignalKind::from_raw(-1);
+ let _ = signal(kind);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
diff --git a/tests/support/leaked_buffers.rs b/tests/support/leaked_buffers.rs
new file mode 100644
index 0000000..a6079fb
--- /dev/null
+++ b/tests/support/leaked_buffers.rs
@@ -0,0 +1,26 @@
+/// Can create buffers of arbitrary lifetime.
+/// Frees created buffers when dropped.
+///
+/// This struct is of course unsafe and the fact that
+/// it must outlive the created slices has to be ensured by
+/// the programmer.
+///
+/// Used at certain test scenarios as a safer version of
+/// Vec::leak, to satisfy the address sanitizer.
+pub struct LeakedBuffers {
+ leaked_vecs: Vec<Box<[u8]>>,
+}
+
+impl LeakedBuffers {
+ pub fn new() -> Self {
+ Self {
+ leaked_vecs: vec![],
+ }
+ }
+ pub unsafe fn create<'a>(&mut self, size: usize) -> &'a mut [u8] {
+ let new_mem = vec![0u8; size].into_boxed_slice();
+ self.leaked_vecs.push(new_mem);
+ let new_mem = self.leaked_vecs.last_mut().unwrap();
+ std::slice::from_raw_parts_mut(new_mem.as_mut_ptr(), new_mem.len())
+ }
+}
diff --git a/tests/support/panic.rs b/tests/support/panic.rs
new file mode 100644
index 0000000..df2f59d
--- /dev/null
+++ b/tests/support/panic.rs
@@ -0,0 +1,34 @@
+use std::panic;
+use std::sync::{Arc, Mutex};
+
+pub fn test_panic<Func: FnOnce() + panic::UnwindSafe>(func: Func) -> Option<String> {
+ static PANIC_MUTEX: Mutex<()> = Mutex::new(());
+
+ {
+ let _guard = PANIC_MUTEX.lock();
+ let panic_file: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
+
+ let prev_hook = panic::take_hook();
+ {
+ let panic_file = panic_file.clone();
+ panic::set_hook(Box::new(move |panic_info| {
+ let panic_location = panic_info.location().unwrap();
+ panic_file
+ .lock()
+ .unwrap()
+ .clone_from(&Some(panic_location.file().to_string()));
+ }));
+ }
+
+ let result = panic::catch_unwind(func);
+ // Return to the previously set panic hook (maybe default) so that we get nice error
+ // messages in the tests.
+ panic::set_hook(prev_hook);
+
+ if result.is_err() {
+ panic_file.lock().unwrap().clone()
+ } else {
+ None
+ }
+ }
+}
diff --git a/tests/sync_barrier.rs b/tests/sync_barrier.rs
index f280fe8..2001c35 100644
--- a/tests/sync_barrier.rs
+++ b/tests/sync_barrier.rs
@@ -1,6 +1,9 @@
#![allow(clippy::unnecessary_operation)]
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
use tokio::sync::Barrier;
diff --git a/tests/sync_broadcast.rs b/tests/sync_broadcast.rs
index 9ef7927..cd66924 100644
--- a/tests/sync_broadcast.rs
+++ b/tests/sync_broadcast.rs
@@ -2,6 +2,9 @@
#![warn(rust_2018_idioms)]
#![cfg(feature = "sync")]
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+
use tokio::sync::broadcast;
use tokio_test::task;
use tokio_test::{
@@ -44,7 +47,7 @@
($e:expr) => {
match assert_err!($e) {
broadcast::error::TryRecvError::Closed => {}
- _ => panic!("did not lag"),
+ _ => panic!("is not closed"),
}
};
}
@@ -273,12 +276,14 @@
#[test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn zero_capacity() {
broadcast::channel::<()>(0);
}
#[test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn capacity_too_big() {
use std::usize;
@@ -286,7 +291,8 @@
}
#[test]
-#[cfg(not(target_os = "android"))]
+#[cfg(panic = "unwind")]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn panic_in_clone() {
use std::panic::{self, AssertUnwindSafe};
@@ -452,6 +458,132 @@
assert_empty!(rx);
}
+#[test]
+fn receiver_len_with_lagged() {
+ let (tx, mut rx) = broadcast::channel(3);
+
+ tx.send(10).unwrap();
+ tx.send(20).unwrap();
+ tx.send(30).unwrap();
+ tx.send(40).unwrap();
+
+ assert_eq!(rx.len(), 4);
+ assert_eq!(assert_recv!(rx), 10);
+
+ tx.send(50).unwrap();
+ tx.send(60).unwrap();
+
+ assert_eq!(rx.len(), 5);
+ assert_lagged!(rx.try_recv(), 1);
+}
+
fn is_closed(err: broadcast::error::RecvError) -> bool {
matches!(err, broadcast::error::RecvError::Closed)
}
+
+#[test]
+fn resubscribe_points_to_tail() {
+ let (tx, mut rx) = broadcast::channel(3);
+ tx.send(1).unwrap();
+
+ let mut rx_resub = rx.resubscribe();
+
+ // verify we're one behind at the start
+ assert_empty!(rx_resub);
+ assert_eq!(assert_recv!(rx), 1);
+
+ // verify we do not affect rx
+ tx.send(2).unwrap();
+ assert_eq!(assert_recv!(rx_resub), 2);
+ tx.send(3).unwrap();
+ assert_eq!(assert_recv!(rx), 2);
+ assert_eq!(assert_recv!(rx), 3);
+ assert_empty!(rx);
+
+ assert_eq!(assert_recv!(rx_resub), 3);
+ assert_empty!(rx_resub);
+}
+
+#[test]
+fn resubscribe_lagged() {
+ let (tx, mut rx) = broadcast::channel(1);
+ tx.send(1).unwrap();
+ tx.send(2).unwrap();
+
+ let mut rx_resub = rx.resubscribe();
+ assert_lagged!(rx.try_recv(), 1);
+ assert_empty!(rx_resub);
+
+ assert_eq!(assert_recv!(rx), 2);
+ assert_empty!(rx);
+ assert_empty!(rx_resub);
+}
+
+#[test]
+fn resubscribe_to_closed_channel() {
+ let (tx, rx) = tokio::sync::broadcast::channel::<u32>(2);
+ drop(tx);
+
+ let mut rx_resub = rx.resubscribe();
+ assert_closed!(rx_resub.try_recv());
+}
+
+#[test]
+fn sender_len() {
+ let (tx, mut rx1) = broadcast::channel(4);
+ let mut rx2 = tx.subscribe();
+
+ assert_eq!(tx.len(), 0);
+ assert!(tx.is_empty());
+
+ tx.send(1).unwrap();
+ tx.send(2).unwrap();
+ tx.send(3).unwrap();
+
+ assert_eq!(tx.len(), 3);
+ assert!(!tx.is_empty());
+
+ assert_recv!(rx1);
+ assert_recv!(rx1);
+
+ assert_eq!(tx.len(), 3);
+ assert!(!tx.is_empty());
+
+ assert_recv!(rx2);
+
+ assert_eq!(tx.len(), 2);
+ assert!(!tx.is_empty());
+
+ tx.send(4).unwrap();
+ tx.send(5).unwrap();
+ tx.send(6).unwrap();
+
+ assert_eq!(tx.len(), 4);
+ assert!(!tx.is_empty());
+}
+
+#[test]
+#[cfg(not(tokio_wasm_not_wasi))]
+fn sender_len_random() {
+ use rand::Rng;
+
+ let (tx, mut rx1) = broadcast::channel(16);
+ let mut rx2 = tx.subscribe();
+
+ for _ in 0..1000 {
+ match rand::thread_rng().gen_range(0..4) {
+ 0 => {
+ let _ = rx1.try_recv();
+ }
+ 1 => {
+ let _ = rx2.try_recv();
+ }
+ _ => {
+ tx.send(0).unwrap();
+ }
+ }
+
+ let expected_len = usize::min(usize::max(rx1.len(), rx2.len()), 16);
+ assert_eq!(tx.len(), expected_len);
+ }
+}
diff --git a/tests/sync_errors.rs b/tests/sync_errors.rs
index 66e8f0c..bf0a6cf 100644
--- a/tests/sync_errors.rs
+++ b/tests/sync_errors.rs
@@ -1,5 +1,8 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
fn is_error<T: std::error::Error + Send + Sync>() {}
diff --git a/tests/sync_mpsc.rs b/tests/sync_mpsc.rs
index 1947d26..6e87096 100644
--- a/tests/sync_mpsc.rs
+++ b/tests/sync_mpsc.rs
@@ -1,18 +1,21 @@
#![allow(clippy::redundant_clone)]
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
-use std::thread;
-use tokio::runtime::Runtime;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+use std::fmt;
+use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::sync::mpsc::error::{TryRecvError, TrySendError};
-use tokio_test::task;
-use tokio_test::{
- assert_err, assert_ok, assert_pending, assert_ready, assert_ready_err, assert_ready_ok,
-};
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
+use tokio_test::*;
-use std::sync::Arc;
-
+#[cfg(not(tokio_wasm))]
mod support {
pub(crate) mod mpsc_stream;
}
@@ -21,7 +24,7 @@
impl AssertSend for mpsc::Sender<i32> {}
impl AssertSend for mpsc::Receiver<i32> {}
-#[tokio::test]
+#[maybe_tokio_test]
async fn send_recv_with_buffer() {
let (tx, mut rx) = mpsc::channel::<i32>(16);
@@ -46,6 +49,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn reserve_disarm() {
let (tx, mut rx) = mpsc::channel::<i32>(2);
let tx1 = tx.clone();
@@ -58,10 +62,10 @@
let permit2 = assert_ok!(tx2.reserve().await);
// But a third should not be ready
- let mut r3 = task::spawn(tx3.reserve());
+ let mut r3 = tokio_test::task::spawn(tx3.reserve());
assert_pending!(r3.poll());
- let mut r4 = task::spawn(tx4.reserve());
+ let mut r4 = tokio_test::task::spawn(tx4.reserve());
assert_pending!(r4.poll());
// Using one of the reserved slots should allow a new handle to become ready
@@ -78,11 +82,12 @@
drop(permit2);
assert!(r4.is_woken());
- let mut r1 = task::spawn(tx1.reserve());
+ let mut r1 = tokio_test::task::spawn(tx1.reserve());
assert_pending!(r1.poll());
}
#[tokio::test]
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
async fn send_recv_stream_with_buffer() {
use tokio_stream::StreamExt;
@@ -100,6 +105,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn async_send_recv_with_buffer() {
let (tx, mut rx) = mpsc::channel(16);
@@ -114,10 +120,11 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn start_send_past_cap() {
use std::future::Future;
- let mut t1 = task::spawn(());
+ let mut t1 = tokio_test::task::spawn(());
let (tx1, mut rx) = mpsc::channel(1);
let tx2 = tx1.clone();
@@ -128,7 +135,7 @@
t1.enter(|cx, _| assert_pending!(r1.as_mut().poll(cx)));
{
- let mut r2 = task::spawn(tx2.reserve());
+ let mut r2 = tokio_test::task::spawn(tx2.reserve());
assert_pending!(r2.poll());
drop(r1);
@@ -147,11 +154,12 @@
#[test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn buffer_gteq_one() {
mpsc::channel::<i32>(0);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn send_recv_unbounded() {
let (tx, mut rx) = mpsc::unbounded_channel::<i32>();
@@ -168,6 +176,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn async_send_recv_unbounded() {
let (tx, mut rx) = mpsc::unbounded_channel();
@@ -182,6 +191,7 @@
}
#[tokio::test]
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
async fn send_recv_stream_unbounded() {
use tokio_stream::StreamExt;
@@ -199,32 +209,32 @@
assert_eq!(None, rx.next().await);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn no_t_bounds_buffer() {
struct NoImpls;
let (tx, mut rx) = mpsc::channel(100);
// sender should be Debug even though T isn't Debug
- println!("{:?}", tx);
+ is_debug(&tx);
// same with Receiver
- println!("{:?}", rx);
+ is_debug(&rx);
// and sender should be Clone even though T isn't Clone
assert!(tx.clone().try_send(NoImpls).is_ok());
assert!(rx.recv().await.is_some());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn no_t_bounds_unbounded() {
struct NoImpls;
let (tx, mut rx) = mpsc::unbounded_channel();
// sender should be Debug even though T isn't Debug
- println!("{:?}", tx);
+ is_debug(&tx);
// same with Receiver
- println!("{:?}", rx);
+ is_debug(&rx);
// and sender should be Clone even though T isn't Clone
assert!(tx.clone().send(NoImpls).is_ok());
@@ -232,6 +242,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn send_recv_buffer_limited() {
let (tx, mut rx) = mpsc::channel::<i32>(1);
@@ -242,7 +253,7 @@
p1.send(1);
// Not ready
- let mut p2 = task::spawn(tx.reserve());
+ let mut p2 = tokio_test::task::spawn(tx.reserve());
assert_pending!(p2.poll());
// Take the value
@@ -261,7 +272,7 @@
assert!(rx.recv().await.is_some());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn recv_close_gets_none_idle() {
let (tx, mut rx) = mpsc::channel::<i32>(10);
@@ -273,12 +284,13 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn recv_close_gets_none_reserved() {
let (tx1, mut rx) = mpsc::channel::<i32>(1);
let tx2 = tx1.clone();
let permit1 = assert_ok!(tx1.reserve().await);
- let mut permit2 = task::spawn(tx2.reserve());
+ let mut permit2 = tokio_test::task::spawn(tx2.reserve());
assert_pending!(permit2.poll());
rx.close();
@@ -287,7 +299,7 @@
assert_ready_err!(permit2.poll());
{
- let mut recv = task::spawn(rx.recv());
+ let mut recv = tokio_test::task::spawn(rx.recv());
assert_pending!(recv.poll());
permit1.send(123);
@@ -300,13 +312,13 @@
assert!(rx.recv().await.is_none());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn tx_close_gets_none() {
let (_, mut rx) = mpsc::channel::<i32>(10);
assert!(rx.recv().await.is_none());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn try_send_fail() {
let (tx, mut rx) = mpsc::channel(1);
@@ -327,7 +339,7 @@
assert!(rx.recv().await.is_none());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn try_send_fail_with_try_recv() {
let (tx, mut rx) = mpsc::channel(1);
@@ -348,7 +360,7 @@
assert_eq!(rx.try_recv(), Err(TryRecvError::Disconnected));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn try_reserve_fails() {
let (tx, mut rx) = mpsc::channel(1);
@@ -372,6 +384,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn drop_permit_releases_permit() {
// poll_ready reserves capacity, ensure that the capacity is released if tx
// is dropped w/o sending a value.
@@ -380,7 +393,7 @@
let permit = assert_ok!(tx1.reserve().await);
- let mut reserve2 = task::spawn(tx2.reserve());
+ let mut reserve2 = tokio_test::task::spawn(tx2.reserve());
assert_pending!(reserve2.poll());
drop(permit);
@@ -389,7 +402,7 @@
assert_ready_ok!(reserve2.poll());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn dropping_rx_closes_channel() {
let (tx, rx) = mpsc::channel(100);
@@ -439,48 +452,57 @@
}
#[test]
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
fn blocking_recv() {
let (tx, mut rx) = mpsc::channel::<u8>(1);
- let sync_code = thread::spawn(move || {
+ let sync_code = std::thread::spawn(move || {
assert_eq!(Some(10), rx.blocking_recv());
});
- Runtime::new().unwrap().block_on(async move {
- let _ = tx.send(10).await;
- });
+ tokio::runtime::Runtime::new()
+ .unwrap()
+ .block_on(async move {
+ let _ = tx.send(10).await;
+ });
sync_code.join().unwrap()
}
#[tokio::test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
async fn blocking_recv_async() {
let (_tx, mut rx) = mpsc::channel::<()>(1);
let _ = rx.blocking_recv();
}
#[test]
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
fn blocking_send() {
let (tx, mut rx) = mpsc::channel::<u8>(1);
- let sync_code = thread::spawn(move || {
+ let sync_code = std::thread::spawn(move || {
tx.blocking_send(10).unwrap();
});
- Runtime::new().unwrap().block_on(async move {
- assert_eq!(Some(10), rx.recv().await);
- });
+ tokio::runtime::Runtime::new()
+ .unwrap()
+ .block_on(async move {
+ assert_eq!(Some(10), rx.recv().await);
+ });
sync_code.join().unwrap()
}
#[tokio::test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
async fn blocking_send_async() {
let (tx, _rx) = mpsc::channel::<()>(1);
let _ = tx.blocking_send(());
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn ready_close_cancel_bounded() {
let (tx, mut rx) = mpsc::channel::<()>(100);
let _tx2 = tx.clone();
@@ -489,7 +511,7 @@
rx.close();
- let mut recv = task::spawn(rx.recv());
+ let mut recv = tokio_test::task::spawn(rx.recv());
assert_pending!(recv.poll());
drop(permit);
@@ -500,13 +522,14 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn permit_available_not_acquired_close() {
let (tx1, mut rx) = mpsc::channel::<()>(1);
let tx2 = tx1.clone();
let permit1 = assert_ok!(tx1.reserve().await);
- let mut permit2 = task::spawn(tx2.reserve());
+ let mut permit2 = tokio_test::task::spawn(tx2.reserve());
assert_pending!(permit2.poll());
rx.close();
@@ -597,3 +620,60 @@
drop(tx);
assert_eq!(Err(TryRecvError::Disconnected), rx.try_recv());
}
+
+#[tokio::test(start_paused = true)]
+#[cfg(feature = "full")]
+async fn recv_timeout() {
+ use tokio::sync::mpsc::error::SendTimeoutError::{Closed, Timeout};
+ use tokio::time::Duration;
+
+ let (tx, rx) = mpsc::channel(5);
+
+ assert_eq!(tx.send_timeout(10, Duration::from_secs(1)).await, Ok(()));
+ assert_eq!(tx.send_timeout(20, Duration::from_secs(1)).await, Ok(()));
+ assert_eq!(tx.send_timeout(30, Duration::from_secs(1)).await, Ok(()));
+ assert_eq!(tx.send_timeout(40, Duration::from_secs(1)).await, Ok(()));
+ assert_eq!(tx.send_timeout(50, Duration::from_secs(1)).await, Ok(()));
+ assert_eq!(
+ tx.send_timeout(60, Duration::from_secs(1)).await,
+ Err(Timeout(60))
+ );
+
+ drop(rx);
+ assert_eq!(
+ tx.send_timeout(70, Duration::from_secs(1)).await,
+ Err(Closed(70))
+ );
+}
+
+#[test]
+#[should_panic = "there is no reactor running, must be called from the context of a Tokio 1.x runtime"]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+fn recv_timeout_panic() {
+ use futures::future::FutureExt;
+ use tokio::time::Duration;
+
+ let (tx, _rx) = mpsc::channel(5);
+ tx.send_timeout(10, Duration::from_secs(1)).now_or_never();
+}
+
+// Tests that channel `capacity` changes and `max_capacity` stays the same
+#[tokio::test]
+async fn test_tx_capacity() {
+ let (tx, _rx) = mpsc::channel::<()>(10);
+ // both capacities are same before
+ assert_eq!(tx.capacity(), 10);
+ assert_eq!(tx.max_capacity(), 10);
+
+ let _permit = tx.reserve().await.unwrap();
+ // after reserve, only capacity should drop by one
+ assert_eq!(tx.capacity(), 9);
+ assert_eq!(tx.max_capacity(), 10);
+
+ tx.send(()).await.unwrap();
+ // after send, capacity should drop by one again
+ assert_eq!(tx.capacity(), 8);
+ assert_eq!(tx.max_capacity(), 10);
+}
+
+fn is_debug<T: fmt::Debug>(_: &T) {}
diff --git a/tests/sync_mpsc_weak.rs b/tests/sync_mpsc_weak.rs
new file mode 100644
index 0000000..0fdfc00
--- /dev/null
+++ b/tests/sync_mpsc_weak.rs
@@ -0,0 +1,513 @@
+#![allow(clippy::redundant_clone)]
+#![warn(rust_2018_idioms)]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering::{Acquire, Release};
+use tokio::sync::mpsc::{self, channel, unbounded_channel};
+use tokio::sync::oneshot;
+
+#[tokio::test]
+async fn weak_sender() {
+ let (tx, mut rx) = channel(11);
+
+ let tx_weak = tokio::spawn(async move {
+ let tx_weak = tx.clone().downgrade();
+
+ for i in 0..10 {
+ if tx.send(i).await.is_err() {
+ return None;
+ }
+ }
+
+ let tx2 = tx_weak
+ .upgrade()
+ .expect("expected to be able to upgrade tx_weak");
+ let _ = tx2.send(20).await;
+ let tx_weak = tx2.downgrade();
+
+ Some(tx_weak)
+ })
+ .await
+ .unwrap();
+
+ for i in 0..12 {
+ let recvd = rx.recv().await;
+
+ match recvd {
+ Some(msg) => {
+ if i == 10 {
+ assert_eq!(msg, 20);
+ }
+ }
+ None => {
+ assert_eq!(i, 11);
+ break;
+ }
+ }
+ }
+
+ let tx_weak = tx_weak.unwrap();
+ let upgraded = tx_weak.upgrade();
+ assert!(upgraded.is_none());
+}
+
+#[tokio::test]
+async fn actor_weak_sender() {
+ pub struct MyActor {
+ receiver: mpsc::Receiver<ActorMessage>,
+ sender: mpsc::WeakSender<ActorMessage>,
+ next_id: u32,
+ pub received_self_msg: bool,
+ }
+
+ enum ActorMessage {
+ GetUniqueId { respond_to: oneshot::Sender<u32> },
+ SelfMessage {},
+ }
+
+ impl MyActor {
+ fn new(
+ receiver: mpsc::Receiver<ActorMessage>,
+ sender: mpsc::WeakSender<ActorMessage>,
+ ) -> Self {
+ MyActor {
+ receiver,
+ sender,
+ next_id: 0,
+ received_self_msg: false,
+ }
+ }
+
+ fn handle_message(&mut self, msg: ActorMessage) {
+ match msg {
+ ActorMessage::GetUniqueId { respond_to } => {
+ self.next_id += 1;
+
+ // The `let _ =` ignores any errors when sending.
+ //
+ // This can happen if the `select!` macro is used
+ // to cancel waiting for the response.
+ let _ = respond_to.send(self.next_id);
+ }
+ ActorMessage::SelfMessage { .. } => {
+ self.received_self_msg = true;
+ }
+ }
+ }
+
+ async fn send_message_to_self(&mut self) {
+ let msg = ActorMessage::SelfMessage {};
+
+ let sender = self.sender.clone();
+
+ // cannot move self.sender here
+ if let Some(sender) = sender.upgrade() {
+ let _ = sender.send(msg).await;
+ self.sender = sender.downgrade();
+ }
+ }
+
+ async fn run(&mut self) {
+ let mut i = 0;
+ while let Some(msg) = self.receiver.recv().await {
+ self.handle_message(msg);
+
+ if i == 0 {
+ self.send_message_to_self().await;
+ }
+
+ i += 1
+ }
+
+ assert!(self.received_self_msg);
+ }
+ }
+
+ #[derive(Clone)]
+ pub struct MyActorHandle {
+ sender: mpsc::Sender<ActorMessage>,
+ }
+
+ impl MyActorHandle {
+ pub fn new() -> (Self, MyActor) {
+ let (sender, receiver) = mpsc::channel(8);
+ let actor = MyActor::new(receiver, sender.clone().downgrade());
+
+ (Self { sender }, actor)
+ }
+
+ pub async fn get_unique_id(&self) -> u32 {
+ let (send, recv) = oneshot::channel();
+ let msg = ActorMessage::GetUniqueId { respond_to: send };
+
+ // Ignore send errors. If this send fails, so does the
+ // recv.await below. There's no reason to check the
+ // failure twice.
+ let _ = self.sender.send(msg).await;
+ recv.await.expect("Actor task has been killed")
+ }
+ }
+
+ let (handle, mut actor) = MyActorHandle::new();
+
+ let actor_handle = tokio::spawn(async move { actor.run().await });
+
+ let _ = tokio::spawn(async move {
+ let _ = handle.get_unique_id().await;
+ drop(handle);
+ })
+ .await;
+
+ let _ = actor_handle.await;
+}
+
+static NUM_DROPPED: AtomicUsize = AtomicUsize::new(0);
+
+#[derive(Debug)]
+struct Msg;
+
+impl Drop for Msg {
+ fn drop(&mut self) {
+ NUM_DROPPED.fetch_add(1, Release);
+ }
+}
+
+// Tests that no pending messages are put onto the channel after `Rx` was
+// dropped.
+//
+// Note: After the introduction of `WeakSender`, which internally
+// used `Arc` and doesn't call a drop of the channel after the last strong
+// `Sender` was dropped while more than one `WeakSender` remains, we want to
+// ensure that no messages are kept in the channel, which were sent after
+// the receiver was dropped.
+#[tokio::test]
+async fn test_msgs_dropped_on_rx_drop() {
+ let (tx, mut rx) = mpsc::channel(3);
+
+ tx.send(Msg {}).await.unwrap();
+ tx.send(Msg {}).await.unwrap();
+
+ // This msg will be pending and should be dropped when `rx` is dropped
+ let sent_fut = tx.send(Msg {});
+
+ let _ = rx.recv().await.unwrap();
+ let _ = rx.recv().await.unwrap();
+
+ sent_fut.await.unwrap();
+
+ drop(rx);
+
+ assert_eq!(NUM_DROPPED.load(Acquire), 3);
+
+ // This msg will not be put onto `Tx` list anymore, since `Rx` is closed.
+ assert!(tx.send(Msg {}).await.is_err());
+
+ assert_eq!(NUM_DROPPED.load(Acquire), 4);
+}
+
+// Tests that a `WeakSender` is upgradeable when other `Sender`s exist.
+#[test]
+fn downgrade_upgrade_sender_success() {
+ let (tx, _rx) = mpsc::channel::<i32>(1);
+ let weak_tx = tx.downgrade();
+ assert!(weak_tx.upgrade().is_some());
+}
+
+// Tests that a `WeakSender` fails to upgrade when no other `Sender` exists.
+#[test]
+fn downgrade_upgrade_sender_failure() {
+ let (tx, _rx) = mpsc::channel::<i32>(1);
+ let weak_tx = tx.downgrade();
+ drop(tx);
+ assert!(weak_tx.upgrade().is_none());
+}
+
+// Tests that a `WeakSender` cannot be upgraded after a `Sender` was dropped,
+// which existed at the time of the `downgrade` call.
+#[test]
+fn downgrade_drop_upgrade() {
+ let (tx, _rx) = mpsc::channel::<i32>(1);
+
+ // the cloned `Tx` is dropped right away
+ let weak_tx = tx.clone().downgrade();
+ drop(tx);
+ assert!(weak_tx.upgrade().is_none());
+}
+
+// Tests that we can upgrade a weak sender with an outstanding permit
+// but no other strong senders.
+#[tokio::test]
+async fn downgrade_get_permit_upgrade_no_senders() {
+ let (tx, _rx) = mpsc::channel::<i32>(1);
+ let weak_tx = tx.downgrade();
+ let _permit = tx.reserve_owned().await.unwrap();
+ assert!(weak_tx.upgrade().is_some());
+}
+
+// Tests that you can downgrade and upgrade a sender with an outstanding permit
+// but no other senders left.
+#[tokio::test]
+async fn downgrade_upgrade_get_permit_no_senders() {
+ let (tx, _rx) = mpsc::channel::<i32>(1);
+ let tx2 = tx.clone();
+ let _permit = tx.reserve_owned().await.unwrap();
+ let weak_tx = tx2.downgrade();
+ drop(tx2);
+ assert!(weak_tx.upgrade().is_some());
+}
+
+// Tests that `downgrade` does not change the `tx_count` of the channel.
+#[test]
+fn test_tx_count_weak_sender() {
+ let (tx, _rx) = mpsc::channel::<i32>(1);
+ let tx_weak = tx.downgrade();
+ let tx_weak2 = tx.downgrade();
+ drop(tx);
+
+ assert!(tx_weak.upgrade().is_none() && tx_weak2.upgrade().is_none());
+}
+
+#[tokio::test]
+async fn weak_unbounded_sender() {
+ let (tx, mut rx) = unbounded_channel();
+
+ let tx_weak = tokio::spawn(async move {
+ let tx_weak = tx.clone().downgrade();
+
+ for i in 0..10 {
+ if tx.send(i).is_err() {
+ return None;
+ }
+ }
+
+ let tx2 = tx_weak
+ .upgrade()
+ .expect("expected to be able to upgrade tx_weak");
+ let _ = tx2.send(20);
+ let tx_weak = tx2.downgrade();
+
+ Some(tx_weak)
+ })
+ .await
+ .unwrap();
+
+ for i in 0..12 {
+ let recvd = rx.recv().await;
+
+ match recvd {
+ Some(msg) => {
+ if i == 10 {
+ assert_eq!(msg, 20);
+ }
+ }
+ None => {
+ assert_eq!(i, 11);
+ break;
+ }
+ }
+ }
+
+ let tx_weak = tx_weak.unwrap();
+ let upgraded = tx_weak.upgrade();
+ assert!(upgraded.is_none());
+}
+
+#[tokio::test]
+async fn actor_weak_unbounded_sender() {
+ pub struct MyActor {
+ receiver: mpsc::UnboundedReceiver<ActorMessage>,
+ sender: mpsc::WeakUnboundedSender<ActorMessage>,
+ next_id: u32,
+ pub received_self_msg: bool,
+ }
+
+ enum ActorMessage {
+ GetUniqueId { respond_to: oneshot::Sender<u32> },
+ SelfMessage {},
+ }
+
+ impl MyActor {
+ fn new(
+ receiver: mpsc::UnboundedReceiver<ActorMessage>,
+ sender: mpsc::WeakUnboundedSender<ActorMessage>,
+ ) -> Self {
+ MyActor {
+ receiver,
+ sender,
+ next_id: 0,
+ received_self_msg: false,
+ }
+ }
+
+ fn handle_message(&mut self, msg: ActorMessage) {
+ match msg {
+ ActorMessage::GetUniqueId { respond_to } => {
+ self.next_id += 1;
+
+ // The `let _ =` ignores any errors when sending.
+ //
+ // This can happen if the `select!` macro is used
+ // to cancel waiting for the response.
+ let _ = respond_to.send(self.next_id);
+ }
+ ActorMessage::SelfMessage { .. } => {
+ self.received_self_msg = true;
+ }
+ }
+ }
+
+ async fn send_message_to_self(&mut self) {
+ let msg = ActorMessage::SelfMessage {};
+
+ let sender = self.sender.clone();
+
+ // cannot move self.sender here
+ if let Some(sender) = sender.upgrade() {
+ let _ = sender.send(msg);
+ self.sender = sender.downgrade();
+ }
+ }
+
+ async fn run(&mut self) {
+ let mut i = 0;
+ while let Some(msg) = self.receiver.recv().await {
+ self.handle_message(msg);
+
+ if i == 0 {
+ self.send_message_to_self().await;
+ }
+
+ i += 1
+ }
+
+ assert!(self.received_self_msg);
+ }
+ }
+
+ #[derive(Clone)]
+ pub struct MyActorHandle {
+ sender: mpsc::UnboundedSender<ActorMessage>,
+ }
+
+ impl MyActorHandle {
+ pub fn new() -> (Self, MyActor) {
+ let (sender, receiver) = mpsc::unbounded_channel();
+ let actor = MyActor::new(receiver, sender.clone().downgrade());
+
+ (Self { sender }, actor)
+ }
+
+ pub async fn get_unique_id(&self) -> u32 {
+ let (send, recv) = oneshot::channel();
+ let msg = ActorMessage::GetUniqueId { respond_to: send };
+
+ // Ignore send errors. If this send fails, so does the
+ // recv.await below. There's no reason to check the
+ // failure twice.
+ let _ = self.sender.send(msg);
+ recv.await.expect("Actor task has been killed")
+ }
+ }
+
+ let (handle, mut actor) = MyActorHandle::new();
+
+ let actor_handle = tokio::spawn(async move { actor.run().await });
+
+ let _ = tokio::spawn(async move {
+ let _ = handle.get_unique_id().await;
+ drop(handle);
+ })
+ .await;
+
+ let _ = actor_handle.await;
+}
+
+static NUM_DROPPED_UNBOUNDED: AtomicUsize = AtomicUsize::new(0);
+
+#[derive(Debug)]
+struct MsgUnbounded;
+
+impl Drop for MsgUnbounded {
+ fn drop(&mut self) {
+ NUM_DROPPED_UNBOUNDED.fetch_add(1, Release);
+ }
+}
+
+// Tests that no pending messages are put onto the channel after `Rx` was
+// dropped.
+//
+// Note: After the introduction of `UnboundedWeakSender`, which internally
+// used `Arc` and doesn't call a drop of the channel after the last strong
+// `UnboundedSender` was dropped while more than one `UnboundedWeakSender`
+// remains, we want to ensure that no messages are kept in the channel, which
+// were sent after the receiver was dropped.
+#[tokio::test]
+async fn test_msgs_dropped_on_unbounded_rx_drop() {
+ let (tx, mut rx) = mpsc::unbounded_channel();
+
+ tx.send(MsgUnbounded {}).unwrap();
+ tx.send(MsgUnbounded {}).unwrap();
+
+ // This msg will be pending and should be dropped when `rx` is dropped
+ let sent = tx.send(MsgUnbounded {});
+
+ let _ = rx.recv().await.unwrap();
+ let _ = rx.recv().await.unwrap();
+
+ sent.unwrap();
+
+ drop(rx);
+
+ assert_eq!(NUM_DROPPED_UNBOUNDED.load(Acquire), 3);
+
+ // This msg will not be put onto `Tx` list anymore, since `Rx` is closed.
+ assert!(tx.send(MsgUnbounded {}).is_err());
+
+ assert_eq!(NUM_DROPPED_UNBOUNDED.load(Acquire), 4);
+}
+
+// Tests that an `WeakUnboundedSender` is upgradeable when other
+// `UnboundedSender`s exist.
+#[test]
+fn downgrade_upgrade_unbounded_sender_success() {
+ let (tx, _rx) = mpsc::unbounded_channel::<i32>();
+ let weak_tx = tx.downgrade();
+ assert!(weak_tx.upgrade().is_some());
+}
+
+// Tests that a `WeakUnboundedSender` fails to upgrade when no other
+// `UnboundedSender` exists.
+#[test]
+fn downgrade_upgrade_unbounded_sender_failure() {
+ let (tx, _rx) = mpsc::unbounded_channel::<i32>();
+ let weak_tx = tx.downgrade();
+ drop(tx);
+ assert!(weak_tx.upgrade().is_none());
+}
+
+// Tests that an `WeakUnboundedSender` cannot be upgraded after an
+// `UnboundedSender` was dropped, which existed at the time of the `downgrade` call.
+#[test]
+fn downgrade_drop_upgrade_unbounded() {
+ let (tx, _rx) = mpsc::unbounded_channel::<i32>();
+
+ // the cloned `Tx` is dropped right away
+ let weak_tx = tx.clone().downgrade();
+ drop(tx);
+ assert!(weak_tx.upgrade().is_none());
+}
+
+// Tests that `downgrade` does not change the `tx_count` of the channel.
+#[test]
+fn test_tx_count_weak_unbounded_sender() {
+ let (tx, _rx) = mpsc::unbounded_channel::<i32>();
+ let tx_weak = tx.downgrade();
+ let tx_weak2 = tx.downgrade();
+ drop(tx);
+
+ assert!(tx_weak.upgrade().is_none() && tx_weak2.upgrade().is_none());
+}
diff --git a/tests/sync_mutex.rs b/tests/sync_mutex.rs
index 090db94..1e35a55 100644
--- a/tests/sync_mutex.rs
+++ b/tests/sync_mutex.rs
@@ -1,13 +1,19 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
use tokio::sync::Mutex;
-use tokio::time::{interval, timeout};
use tokio_test::task::spawn;
use tokio_test::{assert_pending, assert_ready};
use std::sync::Arc;
-use std::time::Duration;
#[test]
fn straight_execution() {
@@ -47,7 +53,7 @@
// But once g unlocks, we can acquire it
drop(g);
assert!(t2.is_woken());
- assert_ready!(t2.poll());
+ let _t2 = assert_ready!(t2.poll());
}
/*
@@ -82,10 +88,14 @@
}
*/
-#[tokio::test]
/// Ensure a mutex is unlocked if a future holding the lock
/// is aborted prematurely.
+#[tokio::test]
+#[cfg(feature = "full")]
async fn aborted_future_1() {
+ use std::time::Duration;
+ use tokio::time::{interval, timeout};
+
let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
{
let m2 = m1.clone();
@@ -93,7 +103,7 @@
timeout(Duration::from_millis(1u64), async move {
let iv = interval(Duration::from_millis(1000));
tokio::pin!(iv);
- m2.lock().await;
+ let _g = m2.lock().await;
iv.as_mut().tick().await;
iv.as_mut().tick().await;
})
@@ -102,16 +112,20 @@
}
// This should succeed as there is no lock left for the mutex.
timeout(Duration::from_millis(1u64), async move {
- m1.lock().await;
+ let _g = m1.lock().await;
})
.await
.expect("Mutex is locked");
}
-#[tokio::test]
/// This test is similar to `aborted_future_1` but this time the
/// aborted future is waiting for the lock.
+#[tokio::test]
+#[cfg(feature = "full")]
async fn aborted_future_2() {
+ use std::time::Duration;
+ use tokio::time::timeout;
+
let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
{
// Lock mutex
@@ -120,7 +134,7 @@
let m2 = m1.clone();
// Try to lock mutex in a future that is aborted prematurely
timeout(Duration::from_millis(1u64), async move {
- m2.lock().await;
+ let _g = m2.lock().await;
})
.await
.unwrap_err();
@@ -128,7 +142,7 @@
}
// This should succeed as there is no lock left for the mutex.
timeout(Duration::from_millis(1u64), async move {
- m1.lock().await;
+ let _g = m1.lock().await;
})
.await
.expect("Mutex is locked");
@@ -141,20 +155,20 @@
let g1 = m.try_lock();
assert!(g1.is_ok());
let g2 = m.try_lock();
- assert!(!g2.is_ok());
+ assert!(g2.is_err());
}
let g3 = m.try_lock();
assert!(g3.is_ok());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn debug_format() {
let s = "debug";
let m = Mutex::new(s.to_string());
assert_eq!(format!("{:?}", s), format!("{:?}", m.lock().await));
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn mutex_debug() {
let s = "data";
let m = Mutex::new(s.to_string());
diff --git a/tests/sync_mutex_owned.rs b/tests/sync_mutex_owned.rs
index 898bf35..ba472fe 100644
--- a/tests/sync_mutex_owned.rs
+++ b/tests/sync_mutex_owned.rs
@@ -1,13 +1,19 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
use tokio::sync::Mutex;
-use tokio::time::{interval, timeout};
use tokio_test::task::spawn;
use tokio_test::{assert_pending, assert_ready};
use std::sync::Arc;
-use std::time::Duration;
#[test]
fn straight_execution() {
@@ -49,10 +55,14 @@
assert_ready!(t2.poll());
}
-#[tokio::test]
/// Ensure a mutex is unlocked if a future holding the lock
/// is aborted prematurely.
+#[tokio::test]
+#[cfg(feature = "full")]
async fn aborted_future_1() {
+ use std::time::Duration;
+ use tokio::time::{interval, timeout};
+
let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
{
let m2 = m1.clone();
@@ -75,10 +85,14 @@
.expect("Mutex is locked");
}
-#[tokio::test]
/// This test is similar to `aborted_future_1` but this time the
/// aborted future is waiting for the lock.
+#[tokio::test]
+#[cfg(feature = "full")]
async fn aborted_future_2() {
+ use std::time::Duration;
+ use tokio::time::timeout;
+
let m1: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
{
// Lock mutex
@@ -108,13 +122,13 @@
let g1 = m.clone().try_lock_owned();
assert!(g1.is_ok());
let g2 = m.clone().try_lock_owned();
- assert!(!g2.is_ok());
+ assert!(g2.is_err());
}
let g3 = m.try_lock_owned();
assert!(g3.is_ok());
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn debug_format() {
let s = "debug";
let m = Arc::new(Mutex::new(s.to_string()));
diff --git a/tests/sync_notify.rs b/tests/sync_notify.rs
index 6c6620b..aa58039 100644
--- a/tests/sync_notify.rs
+++ b/tests/sync_notify.rs
@@ -1,5 +1,8 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
use tokio::sync::Notify;
use tokio_test::task::spawn;
@@ -151,3 +154,74 @@
assert_ready!(notified2.poll());
}
+
+#[test]
+fn test_notify_one_not_enabled() {
+ let notify = Notify::new();
+ let mut future = spawn(notify.notified());
+
+ notify.notify_one();
+ assert_ready!(future.poll());
+}
+
+#[test]
+fn test_notify_one_after_enable() {
+ let notify = Notify::new();
+ let mut future = spawn(notify.notified());
+
+ future.enter(|_, fut| assert!(!fut.enable()));
+
+ notify.notify_one();
+ assert_ready!(future.poll());
+ future.enter(|_, fut| assert!(fut.enable()));
+}
+
+#[test]
+fn test_poll_after_enable() {
+ let notify = Notify::new();
+ let mut future = spawn(notify.notified());
+
+ future.enter(|_, fut| assert!(!fut.enable()));
+ assert_pending!(future.poll());
+}
+
+#[test]
+fn test_enable_after_poll() {
+ let notify = Notify::new();
+ let mut future = spawn(notify.notified());
+
+ assert_pending!(future.poll());
+ future.enter(|_, fut| assert!(!fut.enable()));
+}
+
+#[test]
+fn test_enable_consumes_permit() {
+ let notify = Notify::new();
+
+ // Add a permit.
+ notify.notify_one();
+
+ let mut future1 = spawn(notify.notified());
+ future1.enter(|_, fut| assert!(fut.enable()));
+
+ let mut future2 = spawn(notify.notified());
+ future2.enter(|_, fut| assert!(!fut.enable()));
+}
+
+#[test]
+fn test_waker_update() {
+ use futures::task::noop_waker;
+ use std::future::Future;
+ use std::task::Context;
+
+ let notify = Notify::new();
+ let mut future = spawn(notify.notified());
+
+ let noop = noop_waker();
+ future.enter(|_, fut| assert_pending!(fut.poll(&mut Context::from_waker(&noop))));
+
+ assert_pending!(future.poll());
+ notify.notify_one();
+
+ assert!(future.is_woken());
+}
diff --git a/tests/sync_once_cell.rs b/tests/sync_once_cell.rs
index 18eaf93..38dfa7c 100644
--- a/tests/sync_once_cell.rs
+++ b/tests/sync_once_cell.rs
@@ -4,178 +4,7 @@
use std::mem;
use std::ops::Drop;
use std::sync::atomic::{AtomicU32, Ordering};
-use std::time::Duration;
-use tokio::runtime;
-use tokio::sync::{OnceCell, SetError};
-use tokio::time;
-
-async fn func1() -> u32 {
- 5
-}
-
-async fn func2() -> u32 {
- time::sleep(Duration::from_millis(1)).await;
- 10
-}
-
-async fn func_err() -> Result<u32, ()> {
- Err(())
-}
-
-async fn func_ok() -> Result<u32, ()> {
- Ok(10)
-}
-
-async fn func_panic() -> u32 {
- time::sleep(Duration::from_millis(1)).await;
- panic!();
-}
-
-async fn sleep_and_set() -> u32 {
- // Simulate sleep by pausing time and waiting for another thread to
- // resume clock when calling `set`, then finding the cell being initialized
- // by this call
- time::sleep(Duration::from_millis(2)).await;
- 5
-}
-
-async fn advance_time_and_set(cell: &'static OnceCell<u32>, v: u32) -> Result<(), SetError<u32>> {
- time::advance(Duration::from_millis(1)).await;
- cell.set(v)
-}
-
-#[test]
-fn get_or_init() {
- let rt = runtime::Builder::new_current_thread()
- .enable_time()
- .start_paused(true)
- .build()
- .unwrap();
-
- static ONCE: OnceCell<u32> = OnceCell::const_new();
-
- rt.block_on(async {
- let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
- let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await });
-
- time::advance(Duration::from_millis(1)).await;
- time::resume();
-
- let result1 = handle1.await.unwrap();
- let result2 = handle2.await.unwrap();
-
- assert_eq!(*result1, 5);
- assert_eq!(*result2, 5);
- });
-}
-
-#[test]
-fn get_or_init_panic() {
- let rt = runtime::Builder::new_current_thread()
- .enable_time()
- .build()
- .unwrap();
-
- static ONCE: OnceCell<u32> = OnceCell::const_new();
-
- rt.block_on(async {
- time::pause();
-
- let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
- let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await });
-
- time::advance(Duration::from_millis(1)).await;
-
- let result1 = handle1.await.unwrap();
- let result2 = handle2.await.unwrap();
-
- assert_eq!(*result1, 5);
- assert_eq!(*result2, 5);
- });
-}
-
-#[test]
-fn set_and_get() {
- let rt = runtime::Builder::new_current_thread()
- .enable_time()
- .build()
- .unwrap();
-
- static ONCE: OnceCell<u32> = OnceCell::const_new();
-
- rt.block_on(async {
- let _ = rt.spawn(async { ONCE.set(5) }).await;
- let value = ONCE.get().unwrap();
- assert_eq!(*value, 5);
- });
-}
-
-#[test]
-fn get_uninit() {
- static ONCE: OnceCell<u32> = OnceCell::const_new();
- let uninit = ONCE.get();
- assert!(uninit.is_none());
-}
-
-#[test]
-fn set_twice() {
- static ONCE: OnceCell<u32> = OnceCell::const_new();
-
- let first = ONCE.set(5);
- assert_eq!(first, Ok(()));
- let second = ONCE.set(6);
- assert!(second.err().unwrap().is_already_init_err());
-}
-
-#[test]
-fn set_while_initializing() {
- let rt = runtime::Builder::new_current_thread()
- .enable_time()
- .build()
- .unwrap();
-
- static ONCE: OnceCell<u32> = OnceCell::const_new();
-
- rt.block_on(async {
- time::pause();
-
- let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await });
- let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await });
-
- time::advance(Duration::from_millis(2)).await;
-
- let result1 = handle1.await.unwrap();
- let result2 = handle2.await.unwrap();
-
- assert_eq!(*result1, 5);
- assert!(result2.err().unwrap().is_initializing_err());
- });
-}
-
-#[test]
-fn get_or_try_init() {
- let rt = runtime::Builder::new_current_thread()
- .enable_time()
- .start_paused(true)
- .build()
- .unwrap();
-
- static ONCE: OnceCell<u32> = OnceCell::const_new();
-
- rt.block_on(async {
- let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await });
- let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await });
-
- time::advance(Duration::from_millis(1)).await;
- time::resume();
-
- let result1 = handle1.await.unwrap();
- assert!(result1.is_err());
-
- let result2 = handle2.await.unwrap();
- assert_eq!(*result2.unwrap(), 10);
- });
-}
+use tokio::sync::OnceCell;
#[test]
fn drop_cell() {
@@ -272,3 +101,185 @@
let cell = OnceCell::from(2);
assert_eq!(*cell.get().unwrap(), 2);
}
+
+#[cfg(feature = "parking_lot")]
+mod parking_lot {
+ use super::*;
+
+ use tokio::runtime;
+ use tokio::sync::SetError;
+ use tokio::time;
+
+ use std::time::Duration;
+
+ async fn func1() -> u32 {
+ 5
+ }
+
+ async fn func2() -> u32 {
+ time::sleep(Duration::from_millis(1)).await;
+ 10
+ }
+
+ async fn func_err() -> Result<u32, ()> {
+ Err(())
+ }
+
+ async fn func_ok() -> Result<u32, ()> {
+ Ok(10)
+ }
+
+ async fn func_panic() -> u32 {
+ time::sleep(Duration::from_millis(1)).await;
+ panic!();
+ }
+
+ async fn sleep_and_set() -> u32 {
+ // Simulate sleep by pausing time and waiting for another thread to
+ // resume clock when calling `set`, then finding the cell being initialized
+ // by this call
+ time::sleep(Duration::from_millis(2)).await;
+ 5
+ }
+
+ async fn advance_time_and_set(
+ cell: &'static OnceCell<u32>,
+ v: u32,
+ ) -> Result<(), SetError<u32>> {
+ time::advance(Duration::from_millis(1)).await;
+ cell.set(v)
+ }
+
+ #[test]
+ fn get_or_init() {
+ let rt = runtime::Builder::new_current_thread()
+ .enable_time()
+ .start_paused(true)
+ .build()
+ .unwrap();
+
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+ rt.block_on(async {
+ let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
+ let handle2 = rt.spawn(async { ONCE.get_or_init(func2).await });
+
+ time::advance(Duration::from_millis(1)).await;
+ time::resume();
+
+ let result1 = handle1.await.unwrap();
+ let result2 = handle2.await.unwrap();
+
+ assert_eq!(*result1, 5);
+ assert_eq!(*result2, 5);
+ });
+ }
+
+ #[test]
+ fn get_or_init_panic() {
+ let rt = runtime::Builder::new_current_thread()
+ .enable_time()
+ .build()
+ .unwrap();
+
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+ rt.block_on(async {
+ time::pause();
+
+ let handle1 = rt.spawn(async { ONCE.get_or_init(func1).await });
+ let handle2 = rt.spawn(async { ONCE.get_or_init(func_panic).await });
+
+ time::advance(Duration::from_millis(1)).await;
+
+ let result1 = handle1.await.unwrap();
+ let result2 = handle2.await.unwrap();
+
+ assert_eq!(*result1, 5);
+ assert_eq!(*result2, 5);
+ });
+ }
+
+ #[test]
+ fn set_and_get() {
+ let rt = runtime::Builder::new_current_thread()
+ .enable_time()
+ .build()
+ .unwrap();
+
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+ rt.block_on(async {
+ let _ = rt.spawn(async { ONCE.set(5) }).await;
+ let value = ONCE.get().unwrap();
+ assert_eq!(*value, 5);
+ });
+ }
+
+ #[test]
+ fn get_uninit() {
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+ let uninit = ONCE.get();
+ assert!(uninit.is_none());
+ }
+
+ #[test]
+ fn set_twice() {
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+ let first = ONCE.set(5);
+ assert_eq!(first, Ok(()));
+ let second = ONCE.set(6);
+ assert!(second.err().unwrap().is_already_init_err());
+ }
+
+ #[test]
+ fn set_while_initializing() {
+ let rt = runtime::Builder::new_current_thread()
+ .enable_time()
+ .build()
+ .unwrap();
+
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+ rt.block_on(async {
+ time::pause();
+
+ let handle1 = rt.spawn(async { ONCE.get_or_init(sleep_and_set).await });
+ let handle2 = rt.spawn(async { advance_time_and_set(&ONCE, 10).await });
+
+ time::advance(Duration::from_millis(2)).await;
+
+ let result1 = handle1.await.unwrap();
+ let result2 = handle2.await.unwrap();
+
+ assert_eq!(*result1, 5);
+ assert!(result2.err().unwrap().is_initializing_err());
+ });
+ }
+
+ #[test]
+ fn get_or_try_init() {
+ let rt = runtime::Builder::new_current_thread()
+ .enable_time()
+ .start_paused(true)
+ .build()
+ .unwrap();
+
+ static ONCE: OnceCell<u32> = OnceCell::const_new();
+
+ rt.block_on(async {
+ let handle1 = rt.spawn(async { ONCE.get_or_try_init(func_err).await });
+ let handle2 = rt.spawn(async { ONCE.get_or_try_init(func_ok).await });
+
+ time::advance(Duration::from_millis(1)).await;
+ time::resume();
+
+ let result1 = handle1.await.unwrap();
+ assert!(result1.is_err());
+
+ let result2 = handle2.await.unwrap();
+ assert_eq!(*result2.unwrap(), 10);
+ });
+ }
+}
diff --git a/tests/sync_oneshot.rs b/tests/sync_oneshot.rs
index 1aab810..15b6923 100644
--- a/tests/sync_oneshot.rs
+++ b/tests/sync_oneshot.rs
@@ -1,5 +1,13 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
use tokio::sync::oneshot;
use tokio::sync::oneshot::error::TryRecvError;
@@ -40,7 +48,7 @@
assert_eq!(val, 1);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn async_send_recv() {
let (tx, rx) = oneshot::channel();
@@ -86,6 +94,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn async_rx_closed() {
let (mut tx, rx) = oneshot::channel::<()>();
@@ -170,6 +179,7 @@
#[test]
#[should_panic]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
fn close_try_recv_poll() {
let (_tx, rx) = oneshot::channel::<i32>();
let mut rx = task::spawn(rx);
@@ -203,6 +213,18 @@
}
#[test]
+fn try_recv_after_completion_await() {
+ let (tx, rx) = oneshot::channel::<i32>();
+ let mut rx = task::spawn(rx);
+
+ tx.send(17).unwrap();
+
+ assert_eq!(Ok(17), assert_ready!(rx.poll()));
+ assert_eq!(Err(TryRecvError::Closed), rx.try_recv());
+ rx.close();
+}
+
+#[test]
fn drops_tasks() {
let (mut tx, mut rx) = oneshot::channel::<i32>();
let mut tx_task = task::spawn(());
diff --git a/tests/sync_panic.rs b/tests/sync_panic.rs
new file mode 100644
index 0000000..6c23664
--- /dev/null
+++ b/tests/sync_panic.rs
@@ -0,0 +1,197 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(tokio_wasi)))]
+
+use std::{error::Error, sync::Arc};
+use tokio::{
+ runtime::{Builder, Runtime},
+ sync::{broadcast, mpsc, oneshot, Mutex, RwLock, Semaphore},
+};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+#[test]
+fn broadcast_channel_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let (_, _) = broadcast::channel::<u32>(0);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn mutex_blocking_lock_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ rt.block_on(async {
+ let mutex = Mutex::new(5_u32);
+ let _g = mutex.blocking_lock();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn oneshot_blocking_recv_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ rt.block_on(async {
+ let (_tx, rx) = oneshot::channel::<u8>();
+ let _ = rx.blocking_recv();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn rwlock_with_max_readers_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = RwLock::<u8>::with_max_readers(0, (u32::MAX >> 3) + 1);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn rwlock_blocking_read_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ rt.block_on(async {
+ let lock = RwLock::<u8>::new(0);
+ let _ = lock.blocking_read();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn rwlock_blocking_write_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ rt.block_on(async {
+ let lock = RwLock::<u8>::new(0);
+ let _ = lock.blocking_write();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn mpsc_bounded_channel_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let (_, _) = mpsc::channel::<u8>(0);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn mpsc_bounded_receiver_blocking_recv_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ let (_tx, mut rx) = mpsc::channel::<u8>(1);
+ rt.block_on(async {
+ let _ = rx.blocking_recv();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn mpsc_bounded_sender_blocking_send_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ let (tx, _rx) = mpsc::channel::<u8>(1);
+ rt.block_on(async {
+ let _ = tx.blocking_send(3);
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn mpsc_unbounded_receiver_blocking_recv_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+ let (_tx, mut rx) = mpsc::unbounded_channel::<u8>();
+ rt.block_on(async {
+ let _ = rx.blocking_recv();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn semaphore_merge_unrelated_owned_permits() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let sem1 = Arc::new(Semaphore::new(42));
+ let sem2 = Arc::new(Semaphore::new(42));
+ let mut p1 = sem1.try_acquire_owned().unwrap();
+ let p2 = sem2.try_acquire_owned().unwrap();
+ p1.merge(p2);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn semaphore_merge_unrelated_permits() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let sem1 = Semaphore::new(42);
+ let sem2 = Semaphore::new(42);
+ let mut p1 = sem1.try_acquire().unwrap();
+ let p2 = sem2.try_acquire().unwrap();
+ p1.merge(p2);
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+fn current_thread() -> Runtime {
+ Builder::new_current_thread().enable_all().build().unwrap()
+}
diff --git a/tests/sync_rwlock.rs b/tests/sync_rwlock.rs
index 7d05086..948ec13 100644
--- a/tests/sync_rwlock.rs
+++ b/tests/sync_rwlock.rs
@@ -1,13 +1,19 @@
#![warn(rust_2018_idioms)]
+#![cfg(feature = "sync")]
-use std::sync::Arc;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as maybe_tokio_test;
+
+#[cfg(not(tokio_wasm_not_wasi))]
+use tokio::test as maybe_tokio_test;
+
use std::task::Poll;
use futures::future::FutureExt;
-use futures::stream;
-use futures::stream::StreamExt;
-use tokio::sync::{Barrier, RwLock};
+use tokio::sync::RwLock;
use tokio_test::task::spawn;
use tokio_test::{assert_pending, assert_ready};
@@ -25,7 +31,7 @@
let mut t1 = spawn(rwlock.read());
let _g1 = assert_ready!(t1.poll());
let mut t2 = spawn(rwlock.read());
- assert_ready!(t2.poll());
+ let _g2 = assert_ready!(t2.poll());
}
// When there is an active shared owner, exclusive access should not be possible
@@ -69,7 +75,7 @@
let g2 = reads.pop().unwrap();
drop(g2);
assert!(t1.is_woken());
- assert_ready!(t1.poll());
+ let _g1 = assert_ready!(t1.poll());
}
// When there is an active exclusive owner, subsequent exclusive access should not be possible
@@ -94,7 +100,7 @@
assert_pending!(t2.poll());
drop(g1);
assert!(t2.is_woken());
- assert_ready!(t2.poll());
+ let _g2 = assert_ready!(t2.poll());
}
// when there is an active shared owner, and exclusive access is triggered,
@@ -106,7 +112,7 @@
let _g1 = assert_ready!(t1.poll());
let mut t2 = spawn(rwlock.read());
- assert_ready!(t2.poll());
+ let _g2 = assert_ready!(t2.poll());
let mut t3 = spawn(rwlock.write());
assert_pending!(t3.poll());
@@ -131,11 +137,11 @@
drop(t2);
assert!(t3.is_woken());
- assert_ready!(t3.poll());
+ let _t3 = assert_ready!(t3.poll());
}
// Acquire an RwLock nonexclusively by a single task
-#[tokio::test]
+#[maybe_tokio_test]
async fn read_uncontested() {
let rwlock = RwLock::new(100);
let result = *rwlock.read().await;
@@ -144,7 +150,7 @@
}
// Acquire an uncontested RwLock in exclusive mode
-#[tokio::test]
+#[maybe_tokio_test]
async fn write_uncontested() {
let rwlock = RwLock::new(100);
let mut result = rwlock.write().await;
@@ -153,7 +159,7 @@
}
// RwLocks should be acquired in the order that their Futures are waited upon.
-#[tokio::test]
+#[maybe_tokio_test]
async fn write_order() {
let rwlock = RwLock::<Vec<u32>>::new(vec![]);
let fut2 = rwlock.write().map(|mut guard| guard.push(2));
@@ -166,8 +172,13 @@
}
// A single RwLock is contested by tasks in multiple threads
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn multithreaded() {
+ use futures::stream::{self, StreamExt};
+ use std::sync::Arc;
+ use tokio::sync::Barrier;
+
let barrier = Arc::new(Barrier::new(5));
let rwlock = Arc::new(RwLock::<u32>::new(0));
let rwclone1 = rwlock.clone();
@@ -236,7 +247,7 @@
assert_eq!(*g, 17_000);
}
-#[tokio::test]
+#[maybe_tokio_test]
async fn try_write() {
let lock = RwLock::new(0);
let read_guard = lock.read().await;
diff --git a/tests/sync_semaphore.rs b/tests/sync_semaphore.rs
index a33b878..3d47ed0 100644
--- a/tests/sync_semaphore.rs
+++ b/tests/sync_semaphore.rs
@@ -1,4 +1,7 @@
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
use std::sync::Arc;
use tokio::sync::Semaphore;
@@ -23,6 +26,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn acquire() {
let sem = Arc::new(Semaphore::new(1));
let p1 = sem.try_acquire().unwrap();
@@ -35,6 +39,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn add_permits() {
let sem = Arc::new(Semaphore::new(0));
let sem_clone = sem.clone();
@@ -58,8 +63,34 @@
assert!(sem.try_acquire().is_err());
}
+#[test]
+fn merge() {
+ let sem = Arc::new(Semaphore::new(3));
+ {
+ let mut p1 = sem.try_acquire().unwrap();
+ assert_eq!(sem.available_permits(), 2);
+ let p2 = sem.try_acquire_many(2).unwrap();
+ assert_eq!(sem.available_permits(), 0);
+ p1.merge(p2);
+ assert_eq!(sem.available_permits(), 0);
+ }
+ assert_eq!(sem.available_permits(), 3);
+}
+
+#[test]
+#[cfg(not(tokio_wasm))] // No stack unwinding on wasm targets
+#[should_panic]
+fn merge_unrelated_permits() {
+ let sem1 = Arc::new(Semaphore::new(3));
+ let sem2 = Arc::new(Semaphore::new(3));
+ let mut p1 = sem1.try_acquire().unwrap();
+ let p2 = sem2.try_acquire().unwrap();
+ p1.merge(p2);
+}
+
#[tokio::test]
-async fn stresstest() {
+#[cfg(feature = "full")]
+async fn stress_test() {
let sem = Arc::new(Semaphore::new(5));
let mut join_handles = Vec::new();
for _ in 0..1000 {
@@ -83,13 +114,37 @@
#[test]
fn add_max_amount_permits() {
let s = tokio::sync::Semaphore::new(0);
- s.add_permits(usize::MAX >> 3);
- assert_eq!(s.available_permits(), usize::MAX >> 3);
+ s.add_permits(tokio::sync::Semaphore::MAX_PERMITS);
+ assert_eq!(s.available_permits(), tokio::sync::Semaphore::MAX_PERMITS);
+}
+
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+#[test]
+#[should_panic]
+fn add_more_than_max_amount_permits1() {
+ let s = tokio::sync::Semaphore::new(1);
+ s.add_permits(tokio::sync::Semaphore::MAX_PERMITS);
+}
+
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+#[test]
+#[should_panic]
+fn add_more_than_max_amount_permits2() {
+ let s = Semaphore::new(Semaphore::MAX_PERMITS - 1);
+ s.add_permits(1);
+ s.add_permits(1);
+}
+
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+#[test]
+#[should_panic]
+fn panic_when_exceeds_maxpermits() {
+ let _ = Semaphore::new(Semaphore::MAX_PERMITS + 1);
}
#[test]
-#[should_panic]
-fn add_more_than_max_amount_permits() {
- let s = tokio::sync::Semaphore::new(1);
- s.add_permits(usize::MAX >> 3);
+fn no_panic_at_maxpermits() {
+ let _ = Semaphore::new(Semaphore::MAX_PERMITS);
+ let s = Semaphore::new(Semaphore::MAX_PERMITS - 1);
+ s.add_permits(1);
}
diff --git a/tests/sync_semaphore_owned.rs b/tests/sync_semaphore_owned.rs
index 478c3a3..f694576 100644
--- a/tests/sync_semaphore_owned.rs
+++ b/tests/sync_semaphore_owned.rs
@@ -1,4 +1,7 @@
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
use std::sync::Arc;
use tokio::sync::Semaphore;
@@ -33,6 +36,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn acquire() {
let sem = Arc::new(Semaphore::new(1));
let p1 = sem.clone().try_acquire_owned().unwrap();
@@ -45,6 +49,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn acquire_many() {
let semaphore = Arc::new(Semaphore::new(42));
let permit32 = semaphore.clone().try_acquire_many_owned(32).unwrap();
@@ -60,6 +65,7 @@
}
#[tokio::test]
+#[cfg(feature = "full")]
async fn add_permits() {
let sem = Arc::new(Semaphore::new(0));
let sem_clone = sem.clone();
@@ -83,8 +89,34 @@
assert!(sem.try_acquire_owned().is_err());
}
+#[test]
+fn merge() {
+ let sem = Arc::new(Semaphore::new(3));
+ {
+ let mut p1 = sem.clone().try_acquire_owned().unwrap();
+ assert_eq!(sem.available_permits(), 2);
+ let p2 = sem.clone().try_acquire_many_owned(2).unwrap();
+ assert_eq!(sem.available_permits(), 0);
+ p1.merge(p2);
+ assert_eq!(sem.available_permits(), 0);
+ }
+ assert_eq!(sem.available_permits(), 3);
+}
+
+#[test]
+#[cfg(not(tokio_wasm))] // No stack unwinding on wasm targets
+#[should_panic]
+fn merge_unrelated_permits() {
+ let sem1 = Arc::new(Semaphore::new(3));
+ let sem2 = Arc::new(Semaphore::new(3));
+ let mut p1 = sem1.try_acquire_owned().unwrap();
+ let p2 = sem2.try_acquire_owned().unwrap();
+ p1.merge(p2)
+}
+
#[tokio::test]
-async fn stresstest() {
+#[cfg(feature = "full")]
+async fn stress_test() {
let sem = Arc::new(Semaphore::new(5));
let mut join_handles = Vec::new();
for _ in 0..1000 {
diff --git a/tests/sync_watch.rs b/tests/sync_watch.rs
index b7bbaf7..d4f8ce8 100644
--- a/tests/sync_watch.rs
+++ b/tests/sync_watch.rs
@@ -1,6 +1,9 @@
#![allow(clippy::cognitive_complexity)]
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(feature = "sync")]
+
+#[cfg(tokio_wasm_not_wasi)]
+use wasm_bindgen_test::wasm_bindgen_test as test;
use tokio::sync::watch;
use tokio_test::task::spawn;
@@ -174,17 +177,24 @@
fn borrow_and_update() {
let (tx, mut rx) = watch::channel("one");
+ assert!(!rx.has_changed().unwrap());
+
tx.send("two").unwrap();
+ assert!(rx.has_changed().unwrap());
assert_ready!(spawn(rx.changed()).poll()).unwrap();
assert_pending!(spawn(rx.changed()).poll());
+ assert!(!rx.has_changed().unwrap());
tx.send("three").unwrap();
+ assert!(rx.has_changed().unwrap());
assert_eq!(*rx.borrow_and_update(), "three");
assert_pending!(spawn(rx.changed()).poll());
+ assert!(!rx.has_changed().unwrap());
drop(tx);
assert_eq!(*rx.borrow_and_update(), "three");
assert_ready!(spawn(rx.changed()).poll()).unwrap_err();
+ assert!(rx.has_changed().is_err());
}
#[test]
@@ -201,3 +211,33 @@
drop(rx);
assert!(tx.is_closed());
}
+
+#[test]
+#[cfg(panic = "unwind")]
+#[cfg(not(tokio_wasm))] // wasm currently doesn't support unwinding
+fn send_modify_panic() {
+ let (tx, mut rx) = watch::channel("one");
+
+ tx.send_modify(|old| *old = "two");
+ assert_eq!(*rx.borrow_and_update(), "two");
+
+ let mut rx2 = rx.clone();
+ assert_eq!(*rx2.borrow_and_update(), "two");
+
+ let mut task = spawn(rx2.changed());
+
+ let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+ tx.send_modify(|old| {
+ *old = "panicked";
+ panic!();
+ })
+ }));
+ assert!(result.is_err());
+
+ assert_pending!(task.poll());
+ assert_eq!(*rx.borrow(), "panicked");
+
+ tx.send_modify(|old| *old = "three");
+ assert_ready_ok!(task.poll());
+ assert_eq!(*rx.borrow_and_update(), "three");
+}
diff --git a/tests/task_abort.rs b/tests/task_abort.rs
index 06c61dc..8a1b007 100644
--- a/tests/task_abort.rs
+++ b/tests/task_abort.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery
use std::sync::Arc;
use std::thread::sleep;
@@ -26,10 +26,7 @@
.unwrap();
rt.block_on(async move {
- let handle = tokio::spawn(async move {
- println!("task started");
- tokio::time::sleep(Duration::new(100, 0)).await
- });
+ let handle = tokio::spawn(async move { tokio::time::sleep(Duration::new(100, 0)).await });
// wait for task to sleep.
tokio::time::sleep(Duration::from_millis(10)).await;
@@ -89,7 +86,7 @@
// Note: We do the following to trigger a deferred task cleanup.
//
// The relevant piece of code you want to look at is in:
- // `Inner::block_on` of `basic_scheduler.rs`.
+ // `Inner::block_on` of `scheduler/current_thread.rs`.
//
// We cause the cleanup to happen by having a poll return Pending once
// so that the scheduler can go into the "auxiliary tasks" mode, at
@@ -159,7 +156,6 @@
let handle = tokio::spawn(async move {
// Make sure the Arc is moved into the task
let _notify_dropped = notify_dropped;
- println!("task started");
tokio::time::sleep(Duration::new(100, 0)).await
});
@@ -188,7 +184,6 @@
let handle = tokio::spawn(async move {
// Make sure the Arc is moved into the task
let _panic_dropped = PanicOnDrop;
- println!("task started");
tokio::time::sleep(Duration::new(100, 0)).await
});
@@ -213,7 +208,6 @@
let handle = tokio::spawn(async move {
// Make sure the Arc is moved into the task
let _panic_dropped = PanicOnDrop;
- println!("task started");
tokio::time::sleep(Duration::new(100, 0)).await
});
diff --git a/tests/task_blocking.rs b/tests/task_blocking.rs
index e6cde25..d82a0e0 100644
--- a/tests/task_blocking.rs
+++ b/tests/task_blocking.rs
@@ -1,7 +1,7 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
-use tokio::{runtime, task};
+use tokio::{runtime, task, time};
use tokio_test::assert_ok;
use std::thread;
@@ -77,7 +77,7 @@
#[tokio::test(flavor = "current_thread")]
#[should_panic]
-async fn no_block_in_basic_scheduler() {
+async fn no_block_in_current_thread_scheduler() {
task::block_in_place(|| {});
}
@@ -91,7 +91,7 @@
#[test]
#[should_panic]
-fn no_block_in_basic_block_on() {
+fn no_block_in_current_thread_block_on() {
let rt = runtime::Builder::new_current_thread().build().unwrap();
rt.block_on(async {
task::block_in_place(|| {});
@@ -99,7 +99,7 @@
}
#[test]
-fn can_enter_basic_rt_from_within_block_in_place() {
+fn can_enter_current_thread_rt_from_within_block_in_place() {
let outer = tokio::runtime::Runtime::new().unwrap();
outer.block_on(async {
@@ -227,3 +227,84 @@
done_rx.recv().unwrap().unwrap();
}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn blocking_when_paused() {
+ // Do not auto-advance time when we have started a blocking task that has
+ // not yet finished.
+ time::timeout(
+ Duration::from_secs(3),
+ task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))),
+ )
+ .await
+ .expect("timeout should not trigger")
+ .expect("blocking task should finish");
+
+ // Really: Do not auto-advance time, even if the timeout is short and the
+ // blocking task runs for longer than that. It doesn't matter: Tokio time
+ // is paused; system time is not.
+ time::timeout(
+ Duration::from_millis(1),
+ task::spawn_blocking(|| thread::sleep(Duration::from_millis(50))),
+ )
+ .await
+ .expect("timeout should not trigger")
+ .expect("blocking task should finish");
+}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn blocking_task_wakes_paused_runtime() {
+ let t0 = std::time::Instant::now();
+ time::timeout(
+ Duration::from_secs(15),
+ task::spawn_blocking(|| thread::sleep(Duration::from_millis(1))),
+ )
+ .await
+ .expect("timeout should not trigger")
+ .expect("blocking task should finish");
+ assert!(
+ t0.elapsed() < Duration::from_secs(10),
+ "completing a spawn_blocking should wake the scheduler if it's parked while time is paused"
+ );
+}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn unawaited_blocking_task_wakes_paused_runtime() {
+ let t0 = std::time::Instant::now();
+
+ // When this task finishes, time should auto-advance, even though the
+ // JoinHandle has not been awaited yet.
+ let a = task::spawn_blocking(|| {
+ thread::sleep(Duration::from_millis(1));
+ });
+
+ crate::time::sleep(Duration::from_secs(15)).await;
+ a.await.expect("blocking task should finish");
+ assert!(
+ t0.elapsed() < Duration::from_secs(10),
+ "completing a spawn_blocking should wake the scheduler if it's parked while time is paused"
+ );
+}
+
+#[cfg(feature = "test-util")]
+#[tokio::test(start_paused = true)]
+async fn panicking_blocking_task_wakes_paused_runtime() {
+ let t0 = std::time::Instant::now();
+ let result = time::timeout(
+ Duration::from_secs(15),
+ task::spawn_blocking(|| {
+ thread::sleep(Duration::from_millis(1));
+ panic!("blocking task panicked");
+ }),
+ )
+ .await
+ .expect("timeout should not trigger");
+ assert!(result.is_err(), "blocking task should have panicked");
+ assert!(
+ t0.elapsed() < Duration::from_secs(10),
+ "completing a spawn_blocking should wake the scheduler if it's parked while time is paused"
+ );
+}
diff --git a/tests/task_builder.rs b/tests/task_builder.rs
index 1499abf..78329ff 100644
--- a/tests/task_builder.rs
+++ b/tests/task_builder.rs
@@ -11,6 +11,7 @@
let result = Builder::new()
.name("name")
.spawn(async { "task executed" })
+ .unwrap()
.await;
assert_eq!(result.unwrap(), "task executed");
@@ -21,6 +22,7 @@
let result = Builder::new()
.name("name")
.spawn_blocking(|| "task executed")
+ .unwrap()
.await;
assert_eq!(result.unwrap(), "task executed");
@@ -34,6 +36,7 @@
Builder::new()
.name("name")
.spawn_local(async move { unsend_data })
+ .unwrap()
.await
})
.await;
@@ -43,14 +46,20 @@
#[test]
async fn spawn_without_name() {
- let result = Builder::new().spawn(async { "task executed" }).await;
+ let result = Builder::new()
+ .spawn(async { "task executed" })
+ .unwrap()
+ .await;
assert_eq!(result.unwrap(), "task executed");
}
#[test]
async fn spawn_blocking_without_name() {
- let result = Builder::new().spawn_blocking(|| "task executed").await;
+ let result = Builder::new()
+ .spawn_blocking(|| "task executed")
+ .unwrap()
+ .await;
assert_eq!(result.unwrap(), "task executed");
}
@@ -59,7 +68,12 @@
async fn spawn_local_without_name() {
let unsend_data = Rc::new("task executed");
let result = LocalSet::new()
- .run_until(async move { Builder::new().spawn_local(async move { unsend_data }).await })
+ .run_until(async move {
+ Builder::new()
+ .spawn_local(async move { unsend_data })
+ .unwrap()
+ .await
+ })
.await;
assert_eq!(*result.unwrap(), "task executed");
diff --git a/tests/task_id.rs b/tests/task_id.rs
new file mode 100644
index 0000000..d7b7c0c
--- /dev/null
+++ b/tests/task_id.rs
@@ -0,0 +1,303 @@
+#![warn(rust_2018_idioms)]
+#![allow(clippy::declare_interior_mutable_const)]
+#![cfg(all(feature = "full", tokio_unstable))]
+
+#[cfg(not(tokio_wasi))]
+use std::error::Error;
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+#[cfg(not(tokio_wasi))]
+use tokio::runtime::{Builder, Runtime};
+use tokio::sync::oneshot;
+use tokio::task::{self, Id, LocalSet};
+
+#[cfg(not(tokio_wasi))]
+mod support {
+ pub mod panic;
+}
+#[cfg(not(tokio_wasi))]
+use support::panic::test_panic;
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_spawn() {
+ tokio::spawn(async { println!("task id: {}", task::id()) })
+ .await
+ .unwrap();
+}
+
+#[cfg(not(tokio_wasi))]
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_spawn_blocking() {
+ task::spawn_blocking(|| println!("task id: {}", task::id()))
+ .await
+ .unwrap();
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_collision_current_thread() {
+ let handle1 = tokio::spawn(async { task::id() });
+ let handle2 = tokio::spawn(async { task::id() });
+
+ let (id1, id2) = tokio::join!(handle1, handle2);
+ assert_ne!(id1.unwrap(), id2.unwrap());
+}
+
+#[cfg(not(tokio_wasi))]
+#[tokio::test(flavor = "multi_thread")]
+async fn task_id_collision_multi_thread() {
+ let handle1 = tokio::spawn(async { task::id() });
+ let handle2 = tokio::spawn(async { task::id() });
+
+ let (id1, id2) = tokio::join!(handle1, handle2);
+ assert_ne!(id1.unwrap(), id2.unwrap());
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_ids_match_current_thread() {
+ let (tx, rx) = oneshot::channel();
+ let handle = tokio::spawn(async {
+ let id = rx.await.unwrap();
+ assert_eq!(id, task::id());
+ });
+ tx.send(handle.id()).unwrap();
+ handle.await.unwrap();
+}
+
+#[cfg(not(tokio_wasi))]
+#[tokio::test(flavor = "multi_thread")]
+async fn task_ids_match_multi_thread() {
+ let (tx, rx) = oneshot::channel();
+ let handle = tokio::spawn(async {
+ let id = rx.await.unwrap();
+ assert_eq!(id, task::id());
+ });
+ tx.send(handle.id()).unwrap();
+ handle.await.unwrap();
+}
+
+#[cfg(not(tokio_wasi))]
+#[tokio::test(flavor = "multi_thread")]
+async fn task_id_future_destructor_completion() {
+ struct MyFuture {
+ tx: Option<oneshot::Sender<Id>>,
+ }
+
+ impl Future for MyFuture {
+ type Output = ();
+
+ fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
+ Poll::Ready(())
+ }
+ }
+
+ impl Drop for MyFuture {
+ fn drop(&mut self) {
+ let _ = self.tx.take().unwrap().send(task::id());
+ }
+ }
+
+ let (tx, rx) = oneshot::channel();
+ let handle = tokio::spawn(MyFuture { tx: Some(tx) });
+ let id = handle.id();
+ handle.await.unwrap();
+ assert_eq!(rx.await.unwrap(), id);
+}
+
+#[cfg(not(tokio_wasi))]
+#[tokio::test(flavor = "multi_thread")]
+async fn task_id_future_destructor_abort() {
+ struct MyFuture {
+ tx: Option<oneshot::Sender<Id>>,
+ }
+
+ impl Future for MyFuture {
+ type Output = ();
+
+ fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
+ Poll::Pending
+ }
+ }
+ impl Drop for MyFuture {
+ fn drop(&mut self) {
+ let _ = self.tx.take().unwrap().send(task::id());
+ }
+ }
+
+ let (tx, rx) = oneshot::channel();
+ let handle = tokio::spawn(MyFuture { tx: Some(tx) });
+ let id = handle.id();
+ handle.abort();
+ assert!(handle.await.unwrap_err().is_cancelled());
+ assert_eq!(rx.await.unwrap(), id);
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_output_destructor_handle_dropped_before_completion() {
+ struct MyOutput {
+ tx: Option<oneshot::Sender<Id>>,
+ }
+
+ impl Drop for MyOutput {
+ fn drop(&mut self) {
+ let _ = self.tx.take().unwrap().send(task::id());
+ }
+ }
+
+ struct MyFuture {
+ tx: Option<oneshot::Sender<Id>>,
+ }
+
+ impl Future for MyFuture {
+ type Output = MyOutput;
+
+ fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
+ Poll::Ready(MyOutput { tx: self.tx.take() })
+ }
+ }
+
+ let (tx, mut rx) = oneshot::channel();
+ let handle = tokio::spawn(MyFuture { tx: Some(tx) });
+ let id = handle.id();
+ drop(handle);
+ assert!(rx.try_recv().is_err());
+ assert_eq!(rx.await.unwrap(), id);
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_output_destructor_handle_dropped_after_completion() {
+ struct MyOutput {
+ tx: Option<oneshot::Sender<Id>>,
+ }
+
+ impl Drop for MyOutput {
+ fn drop(&mut self) {
+ let _ = self.tx.take().unwrap().send(task::id());
+ }
+ }
+
+ struct MyFuture {
+ tx_output: Option<oneshot::Sender<Id>>,
+ tx_future: Option<oneshot::Sender<()>>,
+ }
+
+ impl Future for MyFuture {
+ type Output = MyOutput;
+
+ fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {
+ let _ = self.tx_future.take().unwrap().send(());
+ Poll::Ready(MyOutput {
+ tx: self.tx_output.take(),
+ })
+ }
+ }
+
+ let (tx_output, mut rx_output) = oneshot::channel();
+ let (tx_future, rx_future) = oneshot::channel();
+ let handle = tokio::spawn(MyFuture {
+ tx_output: Some(tx_output),
+ tx_future: Some(tx_future),
+ });
+ let id = handle.id();
+ rx_future.await.unwrap();
+ assert!(rx_output.try_recv().is_err());
+ drop(handle);
+ assert_eq!(rx_output.await.unwrap(), id);
+}
+
+#[test]
+fn task_try_id_outside_task() {
+ assert_eq!(None, task::try_id());
+}
+
+#[cfg(not(tokio_wasi))]
+#[test]
+fn task_try_id_inside_block_on() {
+ let rt = Runtime::new().unwrap();
+ rt.block_on(async {
+ assert_eq!(None, task::try_id());
+ });
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_spawn_local() {
+ LocalSet::new()
+ .run_until(async {
+ task::spawn_local(async { println!("task id: {}", task::id()) })
+ .await
+ .unwrap();
+ })
+ .await
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn task_id_nested_spawn_local() {
+ LocalSet::new()
+ .run_until(async {
+ task::spawn_local(async {
+ let parent_id = task::id();
+ LocalSet::new()
+ .run_until(async {
+ task::spawn_local(async move {
+ assert_ne!(parent_id, task::id());
+ })
+ .await
+ .unwrap();
+ })
+ .await;
+ assert_eq!(parent_id, task::id());
+ })
+ .await
+ .unwrap();
+ })
+ .await;
+}
+
+#[cfg(not(tokio_wasi))]
+#[tokio::test(flavor = "multi_thread")]
+async fn task_id_block_in_place_block_on_spawn() {
+ task::spawn(async {
+ let parent_id = task::id();
+
+ task::block_in_place(move || {
+ let rt = Builder::new_current_thread().build().unwrap();
+ rt.block_on(rt.spawn(async move {
+ assert_ne!(parent_id, task::id());
+ }))
+ .unwrap();
+ });
+
+ assert_eq!(parent_id, task::id());
+ })
+ .await
+ .unwrap();
+}
+
+#[cfg(not(tokio_wasi))]
+#[test]
+fn task_id_outside_task_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = task::id();
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[cfg(not(tokio_wasi))]
+#[test]
+fn task_id_inside_block_on_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = Runtime::new().unwrap();
+ rt.block_on(async {
+ task::id();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
diff --git a/tests/task_join_set.rs b/tests/task_join_set.rs
new file mode 100644
index 0000000..b1b6cf9
--- /dev/null
+++ b/tests/task_join_set.rs
@@ -0,0 +1,235 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full"))]
+
+use tokio::sync::oneshot;
+use tokio::task::JoinSet;
+use tokio::time::Duration;
+
+fn rt() -> tokio::runtime::Runtime {
+ tokio::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap()
+}
+
+#[tokio::test(start_paused = true)]
+async fn test_with_sleep() {
+ let mut set = JoinSet::new();
+
+ for i in 0..10 {
+ set.spawn(async move { i });
+ assert_eq!(set.len(), 1 + i);
+ }
+ set.detach_all();
+ assert_eq!(set.len(), 0);
+
+ assert!(matches!(set.join_next().await, None));
+
+ for i in 0..10 {
+ set.spawn(async move {
+ tokio::time::sleep(Duration::from_secs(i as u64)).await;
+ i
+ });
+ assert_eq!(set.len(), 1 + i);
+ }
+
+ let mut seen = [false; 10];
+ while let Some(res) = set.join_next().await.transpose().unwrap() {
+ seen[res] = true;
+ }
+
+ for was_seen in &seen {
+ assert!(was_seen);
+ }
+ assert!(matches!(set.join_next().await, None));
+
+ // Do it again.
+ for i in 0..10 {
+ set.spawn(async move {
+ tokio::time::sleep(Duration::from_secs(i as u64)).await;
+ i
+ });
+ }
+
+ let mut seen = [false; 10];
+ while let Some(res) = set.join_next().await.transpose().unwrap() {
+ seen[res] = true;
+ }
+
+ for was_seen in &seen {
+ assert!(was_seen);
+ }
+ assert!(matches!(set.join_next().await, None));
+}
+
+#[tokio::test]
+async fn test_abort_on_drop() {
+ let mut set = JoinSet::new();
+
+ let mut recvs = Vec::new();
+
+ for _ in 0..16 {
+ let (send, recv) = oneshot::channel::<()>();
+ recvs.push(recv);
+
+ set.spawn(async {
+ // This task will never complete on its own.
+ futures::future::pending::<()>().await;
+ drop(send);
+ });
+ }
+
+ drop(set);
+
+ for recv in recvs {
+ // The task is aborted soon and we will receive an error.
+ assert!(recv.await.is_err());
+ }
+}
+
+#[tokio::test]
+async fn alternating() {
+ let mut set = JoinSet::new();
+
+ assert_eq!(set.len(), 0);
+ set.spawn(async {});
+ assert_eq!(set.len(), 1);
+ set.spawn(async {});
+ assert_eq!(set.len(), 2);
+
+ for _ in 0..16 {
+ let () = set.join_next().await.unwrap().unwrap();
+ assert_eq!(set.len(), 1);
+ set.spawn(async {});
+ assert_eq!(set.len(), 2);
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn abort_tasks() {
+ let mut set = JoinSet::new();
+ let mut num_canceled = 0;
+ let mut num_completed = 0;
+ for i in 0..16 {
+ let abort = set.spawn(async move {
+ tokio::time::sleep(Duration::from_secs(i as u64)).await;
+ i
+ });
+
+ if i % 2 != 0 {
+ // abort odd-numbered tasks.
+ abort.abort();
+ }
+ }
+ loop {
+ match set.join_next().await {
+ Some(Ok(res)) => {
+ num_completed += 1;
+ assert_eq!(res % 2, 0);
+ }
+ Some(Err(e)) => {
+ assert!(e.is_cancelled());
+ num_canceled += 1;
+ }
+ None => break,
+ }
+ }
+
+ assert_eq!(num_canceled, 8);
+ assert_eq!(num_completed, 8);
+}
+
+#[test]
+fn runtime_gone() {
+ let mut set = JoinSet::new();
+ {
+ let rt = rt();
+ set.spawn_on(async { 1 }, rt.handle());
+ drop(rt);
+ }
+
+ assert!(rt()
+ .block_on(set.join_next())
+ .unwrap()
+ .unwrap_err()
+ .is_cancelled());
+}
+
+#[tokio::test(start_paused = true)]
+async fn abort_all() {
+ let mut set: JoinSet<()> = JoinSet::new();
+
+ for _ in 0..5 {
+ set.spawn(futures::future::pending());
+ }
+ for _ in 0..5 {
+ set.spawn(async {
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ });
+ }
+
+ // The join set will now have 5 pending tasks and 5 ready tasks.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+
+ set.abort_all();
+ assert_eq!(set.len(), 10);
+
+ let mut count = 0;
+ while let Some(res) = set.join_next().await {
+ if let Err(err) = res {
+ assert!(err.is_cancelled());
+ }
+ count += 1;
+ }
+ assert_eq!(count, 10);
+ assert_eq!(set.len(), 0);
+}
+
+#[cfg(feature = "parking_lot")]
+mod parking_lot {
+ use super::*;
+
+ use futures::future::FutureExt;
+
+ // This ensures that `join_next` works correctly when the coop budget is
+ // exhausted.
+ #[tokio::test(flavor = "current_thread")]
+ async fn join_set_coop() {
+ // Large enough to trigger coop.
+ const TASK_NUM: u32 = 1000;
+
+ static SEM: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(0);
+
+ let mut set = JoinSet::new();
+
+ for _ in 0..TASK_NUM {
+ set.spawn(async {
+ SEM.add_permits(1);
+ });
+ }
+
+ // Wait for all tasks to complete.
+ //
+ // Since this is a `current_thread` runtime, there's no race condition
+ // between the last permit being added and the task completing.
+ let _ = SEM.acquire_many(TASK_NUM).await.unwrap();
+
+ let mut count = 0;
+ let mut coop_count = 0;
+ loop {
+ match set.join_next().now_or_never() {
+ Some(Some(Ok(()))) => {}
+ Some(Some(Err(err))) => panic!("failed: {}", err),
+ None => {
+ coop_count += 1;
+ tokio::task::yield_now().await;
+ continue;
+ }
+ Some(None) => break,
+ }
+
+ count += 1;
+ }
+ assert!(coop_count >= 1);
+ assert_eq!(count, TASK_NUM);
+ }
+}
diff --git a/tests/task_local.rs b/tests/task_local.rs
index 9981532..949a40c 100644
--- a/tests/task_local.rs
+++ b/tests/task_local.rs
@@ -1,10 +1,17 @@
-tokio::task_local! {
- static REQ_ID: u32;
- pub static FOO: bool;
-}
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
+#![allow(clippy::declare_interior_mutable_const)]
+use std::future::Future;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use tokio::sync::oneshot;
#[tokio::test(flavor = "multi_thread")]
async fn local() {
+ tokio::task_local! {
+ static REQ_ID: u32;
+ pub static FOO: bool;
+ }
+
let j1 = tokio::spawn(REQ_ID.scope(1, async move {
assert_eq!(REQ_ID.get(), 1);
assert_eq!(REQ_ID.get(), 1);
@@ -29,3 +36,84 @@
j2.await.unwrap();
j3.await.unwrap();
}
+
+#[tokio::test]
+async fn task_local_available_on_abort() {
+ tokio::task_local! {
+ static KEY: u32;
+ }
+
+ struct MyFuture {
+ tx_poll: Option<oneshot::Sender<()>>,
+ tx_drop: Option<oneshot::Sender<u32>>,
+ }
+ impl Future for MyFuture {
+ type Output = ();
+
+ fn poll(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
+ if let Some(tx_poll) = self.tx_poll.take() {
+ let _ = tx_poll.send(());
+ }
+ Poll::Pending
+ }
+ }
+ impl Drop for MyFuture {
+ fn drop(&mut self) {
+ let _ = self.tx_drop.take().unwrap().send(KEY.get());
+ }
+ }
+
+ let (tx_drop, rx_drop) = oneshot::channel();
+ let (tx_poll, rx_poll) = oneshot::channel();
+
+ let h = tokio::spawn(KEY.scope(
+ 42,
+ MyFuture {
+ tx_poll: Some(tx_poll),
+ tx_drop: Some(tx_drop),
+ },
+ ));
+
+ rx_poll.await.unwrap();
+ h.abort();
+ assert_eq!(rx_drop.await.unwrap(), 42);
+
+ let err = h.await.unwrap_err();
+ if !err.is_cancelled() {
+ if let Ok(panic) = err.try_into_panic() {
+ std::panic::resume_unwind(panic);
+ } else {
+ panic!();
+ }
+ }
+}
+
+#[tokio::test]
+async fn task_local_available_on_completion_drop() {
+ tokio::task_local! {
+ static KEY: u32;
+ }
+
+ struct MyFuture {
+ tx: Option<oneshot::Sender<u32>>,
+ }
+ impl Future for MyFuture {
+ type Output = ();
+
+ fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
+ Poll::Ready(())
+ }
+ }
+ impl Drop for MyFuture {
+ fn drop(&mut self) {
+ let _ = self.tx.take().unwrap().send(KEY.get());
+ }
+ }
+
+ let (tx, rx) = oneshot::channel();
+
+ let h = tokio::spawn(KEY.scope(42, MyFuture { tx: Some(tx) }));
+
+ assert_eq!(rx.await.unwrap(), 42);
+ h.await.unwrap();
+}
diff --git a/tests/task_local_set.rs b/tests/task_local_set.rs
index f8a35d0..2da87f5 100644
--- a/tests/task_local_set.rs
+++ b/tests/task_local_set.rs
@@ -6,18 +6,23 @@
FutureExt,
};
-use tokio::runtime::{self, Runtime};
+use tokio::runtime;
use tokio::sync::{mpsc, oneshot};
use tokio::task::{self, LocalSet};
use tokio::time;
+#[cfg(not(tokio_wasi))]
use std::cell::Cell;
-use std::sync::atomic::Ordering::{self, SeqCst};
-use std::sync::atomic::{AtomicBool, AtomicUsize};
+use std::sync::atomic::AtomicBool;
+#[cfg(not(tokio_wasi))]
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+#[cfg(not(tokio_wasi))]
+use std::sync::atomic::Ordering::SeqCst;
use std::time::Duration;
#[tokio::test(flavor = "current_thread")]
-async fn local_basic_scheduler() {
+async fn local_current_thread_scheduler() {
LocalSet::new()
.run_until(async {
task::spawn_local(async {}).await.unwrap();
@@ -25,6 +30,7 @@
.await;
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn local_threadpool() {
thread_local! {
@@ -45,6 +51,7 @@
.await;
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn localset_future_threadpool() {
thread_local! {
@@ -60,6 +67,7 @@
local.await;
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn localset_future_timers() {
static RAN1: AtomicBool = AtomicBool::new(false);
@@ -104,6 +112,7 @@
assert!(RAN3.load(Ordering::SeqCst));
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn local_threadpool_timer() {
// This test ensures that runtime services like the timer are properly
@@ -126,7 +135,23 @@
})
.await;
}
+#[test]
+fn enter_guard_spawn() {
+ let local = LocalSet::new();
+ let _guard = local.enter();
+ // Run the local task set.
+ let join = task::spawn_local(async { true });
+ let rt = runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ local.block_on(&rt, async move {
+ assert!(join.await.unwrap());
+ });
+}
+
+#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery
#[test]
// This will panic, since the thread that calls `block_on` cannot use
// in-place blocking inside of `block_on`.
@@ -153,6 +178,7 @@
});
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn local_threadpool_blocking_run() {
thread_local! {
@@ -181,6 +207,7 @@
.await;
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn all_spawns_are_local() {
use futures::future;
@@ -207,6 +234,7 @@
.await;
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread")]
async fn nested_spawn_is_local() {
thread_local! {
@@ -242,6 +270,7 @@
.await;
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
#[test]
fn join_local_future_elsewhere() {
thread_local! {
@@ -255,14 +284,12 @@
local.block_on(&rt, async move {
let (tx, rx) = oneshot::channel();
let join = task::spawn_local(async move {
- println!("hello world running...");
assert!(
ON_RT_THREAD.with(|cell| cell.get()),
"local task must run on local thread, no matter where it is awaited"
);
rx.await.unwrap();
- println!("hello world task done");
"hello world"
});
let join2 = task::spawn(async move {
@@ -272,16 +299,34 @@
);
tx.send(()).expect("task shouldn't have ended yet");
- println!("waking up hello world...");
join.await.expect("task should complete successfully");
-
- println!("hello world task joined");
});
join2.await.unwrap()
});
}
+// Tests for <https://github.com/tokio-rs/tokio/issues/4973>
+#[cfg(not(tokio_wasi))] // Wasi doesn't support threads
+#[tokio::test(flavor = "multi_thread")]
+async fn localset_in_thread_local() {
+ thread_local! {
+ static LOCAL_SET: LocalSet = LocalSet::new();
+ }
+
+ // holds runtime thread until end of main fn.
+ let (_tx, rx) = oneshot::channel::<()>();
+ let handle = tokio::runtime::Handle::current();
+
+ std::thread::spawn(move || {
+ LOCAL_SET.with(|local_set| {
+ handle.block_on(local_set.run_until(async move {
+ let _ = rx.await;
+ }))
+ });
+ });
+}
+
#[test]
fn drop_cancels_tasks() {
use std::rc::Rc;
@@ -345,9 +390,7 @@
),
// Did the test thread panic? We'll find out for sure when we `join`
// with it.
- Err(RecvTimeoutError::Disconnected) => {
- println!("done_rx dropped, did the test thread panic?");
- }
+ Err(RecvTimeoutError::Disconnected) => {}
// Test completed successfully!
Ok(()) => {}
}
@@ -355,6 +398,7 @@
thread.join().expect("test thread should not panic!")
}
+#[cfg_attr(tokio_wasi, ignore = "`unwrap()` in `with_timeout()` panics on Wasi")]
#[test]
fn drop_cancels_remote_tasks() {
// This test reproduces issue #1885.
@@ -377,6 +421,10 @@
});
}
+#[cfg_attr(
+ tokio_wasi,
+ ignore = "FIXME: `task::spawn_local().await.unwrap()` panics on Wasi"
+)]
#[test]
fn local_tasks_wake_join_all() {
// This test reproduces issue #2460.
@@ -398,6 +446,7 @@
});
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery
#[test]
fn local_tasks_are_polled_after_tick() {
// This test depends on timing, so we run it up to five times.
@@ -414,6 +463,7 @@
local_tasks_are_polled_after_tick_inner();
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery
#[tokio::main(flavor = "current_thread")]
async fn local_tasks_are_polled_after_tick_inner() {
// Reproduces issues #1899 and #1900
@@ -450,12 +500,15 @@
tx.send(()).unwrap();
}
- time::sleep(Duration::from_millis(20)).await;
- let rx1 = RX1.load(SeqCst);
- let rx2 = RX2.load(SeqCst);
- println!("EXPECT = {}; RX1 = {}; RX2 = {}", EXPECTED, rx1, rx2);
- assert_eq!(EXPECTED, rx1);
- assert_eq!(EXPECTED, rx2);
+ loop {
+ time::sleep(Duration::from_millis(20)).await;
+ let rx1 = RX1.load(SeqCst);
+ let rx2 = RX2.load(SeqCst);
+
+ if rx1 == EXPECTED && rx2 == EXPECTED {
+ break;
+ }
+ }
});
while let Some(oneshot) = rx.recv().await {
@@ -517,7 +570,122 @@
}
}
-fn rt() -> Runtime {
+#[test]
+fn store_local_set_in_thread_local_with_runtime() {
+ use tokio::runtime::Runtime;
+
+ thread_local! {
+ static CURRENT: RtAndLocalSet = RtAndLocalSet::new();
+ }
+
+ struct RtAndLocalSet {
+ rt: Runtime,
+ local: LocalSet,
+ }
+
+ impl RtAndLocalSet {
+ fn new() -> RtAndLocalSet {
+ RtAndLocalSet {
+ rt: tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap(),
+ local: LocalSet::new(),
+ }
+ }
+
+ async fn inner_method(&self) {
+ self.local
+ .run_until(async move {
+ tokio::task::spawn_local(async {});
+ })
+ .await
+ }
+
+ fn method(&self) {
+ self.rt.block_on(self.inner_method());
+ }
+ }
+
+ CURRENT.with(|f| {
+ f.method();
+ });
+}
+
+#[cfg(tokio_unstable)]
+mod unstable {
+ use tokio::runtime::UnhandledPanic;
+ use tokio::task::LocalSet;
+
+ #[tokio::test]
+ #[should_panic(
+ expected = "a spawned task panicked and the LocalSet is configured to shutdown on unhandled panic"
+ )]
+ async fn shutdown_on_panic() {
+ LocalSet::new()
+ .unhandled_panic(UnhandledPanic::ShutdownRuntime)
+ .run_until(async {
+ tokio::task::spawn_local(async {
+ panic!("boom");
+ });
+
+ futures::future::pending::<()>().await;
+ })
+ .await;
+ }
+
+ // This test compares that, when the task driving `run_until` has already
+ // consumed budget, the `run_until` future has less budget than a "spawned"
+ // task.
+ //
+ // "Budget" is a fuzzy metric as the Tokio runtime is able to change values
+ // internally. This is why the test uses indirection to test this.
+ #[tokio::test]
+ async fn run_until_does_not_get_own_budget() {
+ // Consume some budget
+ tokio::task::consume_budget().await;
+
+ LocalSet::new()
+ .run_until(async {
+ let spawned = tokio::spawn(async {
+ let mut spawned_n = 0;
+
+ {
+ let mut spawned = tokio_test::task::spawn(async {
+ loop {
+ spawned_n += 1;
+ tokio::task::consume_budget().await;
+ }
+ });
+ // Poll once
+ assert!(!spawned.poll().is_ready());
+ }
+
+ spawned_n
+ });
+
+ let mut run_until_n = 0;
+ {
+ let mut run_until = tokio_test::task::spawn(async {
+ loop {
+ run_until_n += 1;
+ tokio::task::consume_budget().await;
+ }
+ });
+ // Poll once
+ assert!(!run_until.poll().is_ready());
+ }
+
+ let spawned_n = spawned.await.unwrap();
+ assert_ne!(spawned_n, 0);
+ assert_ne!(run_until_n, 0);
+ assert!(spawned_n > run_until_n);
+ })
+ .await
+ }
+}
+
+fn rt() -> runtime::Runtime {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
diff --git a/tests/task_panic.rs b/tests/task_panic.rs
new file mode 100644
index 0000000..e4cedce
--- /dev/null
+++ b/tests/task_panic.rs
@@ -0,0 +1,123 @@
+#![warn(rust_2018_idioms)]
+#![allow(clippy::declare_interior_mutable_const)]
+#![cfg(all(feature = "full", not(tokio_wasi)))]
+
+use futures::future;
+use std::error::Error;
+use tokio::runtime::Builder;
+use tokio::task::{self, block_in_place};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+#[test]
+fn block_in_place_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = Builder::new_current_thread().enable_all().build().unwrap();
+ rt.block_on(async {
+ block_in_place(|| {});
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn local_set_spawn_local_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _local = task::LocalSet::new();
+
+ let _ = task::spawn_local(async {});
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn local_set_block_on_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = Builder::new_current_thread().enable_all().build().unwrap();
+ let local = task::LocalSet::new();
+
+ rt.block_on(async {
+ local.block_on(&rt, future::pending::<()>());
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn spawn_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ tokio::spawn(future::pending::<()>());
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn local_key_sync_scope_panic_caller() -> Result<(), Box<dyn Error>> {
+ tokio::task_local! {
+ static NUMBER: u32;
+ }
+
+ let panic_location_file = test_panic(|| {
+ NUMBER.sync_scope(1, || {
+ NUMBER.with(|_| {
+ NUMBER.sync_scope(1, || {});
+ });
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn local_key_with_panic_caller() -> Result<(), Box<dyn Error>> {
+ tokio::task_local! {
+ static NUMBER: u32;
+ }
+
+ let panic_location_file = test_panic(|| {
+ NUMBER.with(|_| {});
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn local_key_get_panic_caller() -> Result<(), Box<dyn Error>> {
+ tokio::task_local! {
+ static NUMBER: u32;
+ }
+
+ let panic_location_file = test_panic(|| {
+ NUMBER.get();
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
diff --git a/tests/tcp_accept.rs b/tests/tcp_accept.rs
index 5ffb946..ba4a498 100644
--- a/tests/tcp_accept.rs
+++ b/tests/tcp_accept.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{mpsc, oneshot};
diff --git a/tests/tcp_connect.rs b/tests/tcp_connect.rs
index cbe68fa..f238442 100644
--- a/tests/tcp_connect.rs
+++ b/tests/tcp_connect.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::oneshot;
@@ -35,6 +35,7 @@
}
#[tokio::test]
+#[cfg(not(tokio_no_ipv6))]
async fn connect_v6() {
let srv = assert_ok!(TcpListener::bind("[::1]:0").await);
let addr = assert_ok!(srv.local_addr());
diff --git a/tests/tcp_echo.rs b/tests/tcp_echo.rs
index 5bb7ff0..f47d4ac 100644
--- a/tests/tcp_echo.rs
+++ b/tests/tcp_echo.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use tokio::io::{self, AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
diff --git a/tests/tcp_into_split.rs b/tests/tcp_into_split.rs
index 2e06643..010984a 100644
--- a/tests/tcp_into_split.rs
+++ b/tests/tcp_into_split.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use std::io::{Error, ErrorKind, Result};
use std::io::{Read, Write};
diff --git a/tests/tcp_into_std.rs b/tests/tcp_into_std.rs
index 4bf24c1..320d994 100644
--- a/tests/tcp_into_std.rs
+++ b/tests/tcp_into_std.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use std::io::Read;
use std::io::Result;
diff --git a/tests/tcp_peek.rs b/tests/tcp_peek.rs
index aecc0ac..b712023 100644
--- a/tests/tcp_peek.rs
+++ b/tests/tcp_peek.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use tokio::io::AsyncReadExt;
use tokio::net::TcpStream;
@@ -15,7 +15,7 @@
let addr = listener.local_addr().unwrap();
let t = thread::spawn(move || assert_ok!(listener.accept()).0);
- let left = net::TcpStream::connect(&addr).unwrap();
+ let left = net::TcpStream::connect(addr).unwrap();
let mut right = t.join().unwrap();
let _ = right.write(&[1, 2, 3, 4]).unwrap();
diff --git a/tests/tcp_shutdown.rs b/tests/tcp_shutdown.rs
index 536a161..1d477f3 100644
--- a/tests/tcp_shutdown.rs
+++ b/tests/tcp_shutdown.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use tokio::io::{self, AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
diff --git a/tests/tcp_socket.rs b/tests/tcp_socket.rs
index 9258864..66309d3 100644
--- a/tests/tcp_socket.rs
+++ b/tests/tcp_socket.rs
@@ -1,6 +1,7 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
+use std::time::Duration;
use tokio::net::TcpSocket;
use tokio_test::assert_ok;
@@ -23,6 +24,7 @@
}
#[tokio::test]
+#[cfg(not(tokio_no_ipv6))]
async fn basic_usage_v6() {
// Create server
let addr = assert_ok!("[::1]:0".parse());
@@ -58,3 +60,16 @@
// Accept
let _ = assert_ok!(srv.accept().await);
}
+
+#[tokio::test]
+async fn basic_linger() {
+ // Create server
+ let addr = assert_ok!("127.0.0.1:0".parse());
+ let srv = assert_ok!(TcpSocket::new_v4());
+ assert_ok!(srv.bind(addr));
+
+ assert!(srv.linger().unwrap().is_none());
+
+ srv.set_linger(Some(Duration::new(0, 0))).unwrap();
+ assert_eq!(srv.linger().unwrap(), Some(Duration::new(0, 0)));
+}
diff --git a/tests/tcp_split.rs b/tests/tcp_split.rs
index 7171dac..335b21f 100644
--- a/tests/tcp_split.rs
+++ b/tests/tcp_split.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use std::io::Result;
use std::io::{Read, Write};
@@ -36,7 +36,7 @@
assert_eq!(peek_len1, read_len);
assert_eq!(&read_buf[..read_len], MSG);
- write_half.write(MSG).await?;
+ assert_eq!(write_half.write(MSG).await?, MSG.len());
handle.join().unwrap();
Ok(())
}
diff --git a/tests/tcp_stream.rs b/tests/tcp_stream.rs
index 0b5d12a..31fe3ba 100644
--- a/tests/tcp_stream.rs
+++ b/tests/tcp_stream.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support bind
use tokio::io::{AsyncReadExt, AsyncWriteExt, Interest};
use tokio::net::{TcpListener, TcpStream};
@@ -254,30 +254,34 @@
(client, server)
}
-fn read_until_pending(stream: &mut TcpStream) {
+fn read_until_pending(stream: &mut TcpStream) -> usize {
let mut buf = vec![0u8; 1024 * 1024];
+ let mut total = 0;
loop {
match stream.try_read(&mut buf) {
- Ok(_) => (),
+ Ok(n) => total += n,
Err(err) => {
assert_eq!(err.kind(), io::ErrorKind::WouldBlock);
break;
}
}
}
+ total
}
-fn write_until_pending(stream: &mut TcpStream) {
+fn write_until_pending(stream: &mut TcpStream) -> usize {
let buf = vec![0u8; 1024 * 1024];
+ let mut total = 0;
loop {
match stream.try_write(&buf) {
- Ok(_) => (),
+ Ok(n) => total += n,
Err(err) => {
assert_eq!(err.kind(), io::ErrorKind::WouldBlock);
break;
}
}
}
+ total
}
#[tokio::test]
@@ -357,3 +361,40 @@
}
}
}
+
+// read_closed is a best effort event, so test only for no false positives.
+#[tokio::test]
+async fn read_closed() {
+ let (client, mut server) = create_pair().await;
+
+ let mut ready_fut = task::spawn(client.ready(Interest::READABLE));
+ assert_pending!(ready_fut.poll());
+
+ assert_ok!(server.write_all(b"ping").await);
+
+ let ready_event = assert_ok!(ready_fut.await);
+
+ assert!(!ready_event.is_read_closed());
+}
+
+// write_closed is a best effort event, so test only for no false positives.
+#[tokio::test]
+async fn write_closed() {
+ let (mut client, mut server) = create_pair().await;
+
+ // Fill the write buffer.
+ let write_size = write_until_pending(&mut client);
+ let mut ready_fut = task::spawn(client.ready(Interest::WRITABLE));
+ assert_pending!(ready_fut.poll());
+
+ // Drain the socket to make client writable.
+ let mut read_size = 0;
+ while read_size < write_size {
+ server.readable().await.unwrap();
+ read_size += read_until_pending(&mut server);
+ }
+
+ let ready_event = assert_ok!(ready_fut.await);
+
+ assert!(!ready_event.is_write_closed());
+}
diff --git a/tests/time_interval.rs b/tests/time_interval.rs
index 5f7bf55..186582e 100644
--- a/tests/time_interval.rs
+++ b/tests/time_interval.rs
@@ -166,6 +166,42 @@
check_interval_poll!(i, start, 1800);
}
+#[tokio::test(start_paused = true)]
+async fn reset() {
+ let start = Instant::now();
+
+ // This is necessary because the timer is only so granular, and in order for
+ // all our ticks to resolve, the time needs to be 1ms ahead of what we
+ // expect, so that the runtime will see that it is time to resolve the timer
+ time::advance(ms(1)).await;
+
+ let mut i = task::spawn(time::interval_at(start, ms(300)));
+
+ check_interval_poll!(i, start, 0);
+
+ time::advance(ms(100)).await;
+ check_interval_poll!(i, start);
+
+ time::advance(ms(200)).await;
+ check_interval_poll!(i, start, 300);
+
+ time::advance(ms(100)).await;
+ check_interval_poll!(i, start);
+
+ i.reset();
+
+ time::advance(ms(250)).await;
+ check_interval_poll!(i, start);
+
+ time::advance(ms(50)).await;
+ // We add one because when using `reset` method, `Interval` adds the
+ // `period` from `Instant::now()`, which will always be off by one
+ check_interval_poll!(i, start, 701);
+
+ time::advance(ms(300)).await;
+ check_interval_poll!(i, start, 1001);
+}
+
fn poll_next(interval: &mut task::Spawn<time::Interval>) -> Poll<Instant> {
interval.enter(|cx, mut interval| interval.poll_tick(cx))
}
diff --git a/tests/time_panic.rs b/tests/time_panic.rs
new file mode 100644
index 0000000..aaff11b
--- /dev/null
+++ b/tests/time_panic.rs
@@ -0,0 +1,93 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support panic recovery
+
+use futures::future;
+use std::error::Error;
+use std::time::Duration;
+use tokio::runtime::{Builder, Runtime};
+use tokio::time::{self, interval, interval_at, timeout, Instant};
+
+mod support {
+ pub mod panic;
+}
+use support::panic::test_panic;
+
+#[test]
+fn pause_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+
+ rt.block_on(async {
+ time::pause();
+ time::pause();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn resume_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let rt = current_thread();
+
+ rt.block_on(async {
+ time::resume();
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn interval_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = interval(Duration::from_millis(0));
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn interval_at_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ let _ = interval_at(Instant::now(), Duration::from_millis(0));
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+#[test]
+fn timeout_panic_caller() -> Result<(), Box<dyn Error>> {
+ let panic_location_file = test_panic(|| {
+ // Runtime without `enable_time` so it has no current timer set.
+ let rt = Builder::new_current_thread().build().unwrap();
+ rt.block_on(async {
+ let _ = timeout(Duration::from_millis(5), future::pending::<()>());
+ });
+ });
+
+ // The panic location should be in this file
+ assert_eq!(&panic_location_file.unwrap(), file!());
+
+ Ok(())
+}
+
+fn current_thread() -> Runtime {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+}
diff --git a/tests/time_pause.rs b/tests/time_pause.rs
index 02e050a..c772e38 100644
--- a/tests/time_pause.rs
+++ b/tests/time_pause.rs
@@ -4,7 +4,10 @@
use rand::SeedableRng;
use rand::{rngs::StdRng, Rng};
use tokio::time::{self, Duration, Instant, Sleep};
-use tokio_test::{assert_elapsed, assert_err, assert_pending, assert_ready, assert_ready_eq, task};
+use tokio_test::{assert_elapsed, assert_pending, assert_ready, assert_ready_eq, task};
+
+#[cfg(not(tokio_wasi))]
+use tokio_test::assert_err;
use std::{
future::Future,
@@ -26,12 +29,14 @@
t.await.unwrap();
}
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[should_panic]
async fn pause_time_in_main_threads() {
tokio::time::pause();
}
+#[cfg(all(feature = "full", not(tokio_wasi)))] // Wasi doesn't support threads
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
async fn pause_time_in_spawn_threads() {
let t = tokio::spawn(async {
diff --git a/tests/time_rt.rs b/tests/time_rt.rs
index 23367be..20f9e18 100644
--- a/tests/time_rt.rs
+++ b/tests/time_rt.rs
@@ -5,6 +5,7 @@
use std::sync::mpsc;
+#[cfg(all(feature = "rt-multi-thread", not(tokio_wasi)))] // Wasi doesn't support threads
#[test]
fn timer_with_threaded_runtime() {
use tokio::runtime::Runtime;
@@ -25,7 +26,7 @@
}
#[test]
-fn timer_with_basic_scheduler() {
+fn timer_with_current_thread_scheduler() {
use tokio::runtime::Builder;
let rt = Builder::new_current_thread().enable_all().build().unwrap();
diff --git a/tests/time_sleep.rs b/tests/time_sleep.rs
index 20477d2..4174a73 100644
--- a/tests/time_sleep.rs
+++ b/tests/time_sleep.rs
@@ -168,6 +168,7 @@
assert_ready!(sleep.poll());
}
+#[cfg(not(tokio_wasi))] // Wasi doesn't support panic recovery
#[test]
#[should_panic]
fn creating_sleep_outside_of_context() {
@@ -188,10 +189,7 @@
#[tokio::test]
async fn short_sleeps() {
- for i in 0..10000 {
- if (i % 10) == 0 {
- eprintln!("=== {}", i);
- }
+ for _ in 0..10000 {
tokio::time::sleep(std::time::Duration::from_millis(0)).await;
}
}
@@ -236,22 +234,6 @@
}
#[tokio::test]
-#[should_panic(expected = "Duration too far into the future")]
-async fn very_long_sleeps() {
- tokio::time::pause();
-
- // Some platforms (eg macos) can't represent times this far in the future
- if let Some(deadline) = tokio::time::Instant::now().checked_add(Duration::from_secs(1u64 << 62))
- {
- tokio::time::sleep_until(deadline).await;
- } else {
- // make it pass anyway (we can't skip/ignore the test based on the
- // result of checked_add)
- panic!("Duration too far into the future (test ignored)")
- }
-}
-
-#[tokio::test]
async fn reset_after_firing() {
let timer = tokio::time::sleep(std::time::Duration::from_millis(1));
tokio::pin!(timer);
@@ -333,18 +315,18 @@
tokio::time::pause();
- let mut lock = list.lock().unwrap();
+ {
+ let mut lock = list.lock().unwrap();
- for _ in 0..100 {
- let mut timer = Box::pin(tokio::time::sleep(Duration::from_millis(10)));
+ for _ in 0..100 {
+ let mut timer = Box::pin(tokio::time::sleep(Duration::from_millis(10)));
- let _ = timer.as_mut().poll(&mut Context::from_waker(&arc_wake));
+ let _ = timer.as_mut().poll(&mut Context::from_waker(&arc_wake));
- lock.push(timer);
+ lock.push(timer);
+ }
}
- drop(lock);
-
tokio::time::sleep(Duration::from_millis(11)).await;
assert!(
diff --git a/tests/time_timeout.rs b/tests/time_timeout.rs
index dbd80eb..be6b8fb 100644
--- a/tests/time_timeout.rs
+++ b/tests/time_timeout.rs
@@ -17,6 +17,7 @@
assert_ready_ok!(fut.poll());
}
+#[cfg_attr(tokio_wasi, ignore = "FIXME: `fut.poll()` panics on Wasi")]
#[tokio::test]
async fn completed_future_past_deadline() {
// Wrap it with a deadline
@@ -135,3 +136,16 @@
fn ms(n: u64) -> Duration {
Duration::from_millis(n)
}
+
+#[tokio::test]
+async fn timeout_is_not_exhausted_by_future() {
+ let fut = timeout(ms(1), async {
+ let mut buffer = [0u8; 1];
+ loop {
+ use tokio::io::AsyncReadExt;
+ let _ = tokio::io::empty().read(&mut buffer).await;
+ }
+ });
+
+ assert!(fut.await.is_err());
+}
diff --git a/tests/udp.rs b/tests/udp.rs
index ec2a1e9..2b6ab4d 100644
--- a/tests/udp.rs
+++ b/tests/udp.rs
@@ -1,5 +1,5 @@
#![warn(rust_2018_idioms)]
-#![cfg(feature = "full")]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support bind or UDP
use futures::future::poll_fn;
use std::io;
@@ -39,7 +39,7 @@
let mut recv_buf = [0u8; 32];
let mut read = ReadBuf::new(&mut recv_buf);
- let _len = poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?;
+ poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?;
assert_eq!(read.filled(), MSG);
Ok(())
diff --git a/tests/uds_datagram.rs b/tests/uds_datagram.rs
index 5e5486b..c08bd45 100644
--- a/tests/uds_datagram.rs
+++ b/tests/uds_datagram.rs
@@ -29,9 +29,7 @@
let server_socket = UnixDatagram::bind(server_path.clone())?;
tokio::spawn(async move {
- if let Err(e) = echo_server(server_socket).await {
- eprintln!("Error in echo server: {}", e);
- }
+ let _ = echo_server(server_socket).await;
});
{
@@ -55,9 +53,7 @@
let server_socket = UnixDatagram::bind(server_path.clone())?;
tokio::spawn(async move {
- if let Err(e) = echo_server(server_socket).await {
- eprintln!("Error in echo server: {}", e);
- }
+ let _ = echo_server(server_socket).await;
});
{
@@ -181,7 +177,7 @@
let mut recv_buf = [0u8; 32];
let mut read = ReadBuf::new(&mut recv_buf);
- let _len = poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?;
+ poll_fn(|cx| receiver.poll_recv(cx, &mut read)).await?;
assert_eq!(read.filled(), msg);
Ok(())
diff --git a/tests/uds_stream.rs b/tests/uds_stream.rs
index 5f1b4cf..b8c4e6a 100644
--- a/tests/uds_stream.rs
+++ b/tests/uds_stream.rs
@@ -25,13 +25,13 @@
let connect = UnixStream::connect(&sock_path);
let ((mut server, _), mut client) = try_join(accept, connect).await?;
- // Write to the client. TODO: Switch to write_all.
- let write_len = client.write(b"hello").await?;
- assert_eq!(write_len, 5);
+ // Write to the client.
+ client.write_all(b"hello").await?;
drop(client);
- // Read from the server. TODO: Switch to read_to_end.
- let mut buf = [0u8; 5];
- server.read_exact(&mut buf).await?;
+
+ // Read from the server.
+ let mut buf = vec![];
+ server.read_to_end(&mut buf).await?;
assert_eq!(&buf, b"hello");
let len = server.read(&mut buf).await?;
assert_eq!(len, 0);
diff --git a/tests/unwindsafe.rs b/tests/unwindsafe.rs
new file mode 100644
index 0000000..3e63820
--- /dev/null
+++ b/tests/unwindsafe.rs
@@ -0,0 +1,42 @@
+#![warn(rust_2018_idioms)]
+#![cfg(all(feature = "full", not(tokio_wasi)))] // Wasi does not support panic recovery
+
+use std::panic::{RefUnwindSafe, UnwindSafe};
+
+#[test]
+fn notify_is_unwind_safe() {
+ is_unwind_safe::<tokio::sync::Notify>();
+}
+
+#[test]
+fn join_handle_is_unwind_safe() {
+ is_unwind_safe::<tokio::task::JoinHandle<()>>();
+}
+
+#[test]
+fn net_types_are_unwind_safe() {
+ is_unwind_safe::<tokio::net::TcpListener>();
+ is_unwind_safe::<tokio::net::TcpSocket>();
+ is_unwind_safe::<tokio::net::TcpStream>();
+ is_unwind_safe::<tokio::net::UdpSocket>();
+}
+
+#[test]
+#[cfg(unix)]
+fn unix_net_types_are_unwind_safe() {
+ is_unwind_safe::<tokio::net::UnixDatagram>();
+ is_unwind_safe::<tokio::net::UnixListener>();
+ is_unwind_safe::<tokio::net::UnixStream>();
+}
+
+#[test]
+#[cfg(windows)]
+fn windows_net_types_are_unwind_safe() {
+ use tokio::net::windows::named_pipe::NamedPipeClient;
+ use tokio::net::windows::named_pipe::NamedPipeServer;
+
+ is_unwind_safe::<NamedPipeClient>();
+ is_unwind_safe::<NamedPipeServer>();
+}
+
+fn is_unwind_safe<T: UnwindSafe + RefUnwindSafe>() {}
