Skip to content
forked from rust-lang/rust
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit fe05246

Browse files
committedMar 6, 2024
Less syscalls for the copy_file_range probe
If it's obvious from the actual syscall results themselves that the syscall is supported or unsupported, don't do an extra syscall with an invalid file descriptor. CC rust-lang#122052
1 parent a27191d commit fe05246

File tree

1 file changed

+48
-23
lines changed

1 file changed

+48
-23
lines changed
 

‎std/src/sys/pal/unix/kernel_copy.rs

+48-23
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
560560
// We store the availability in a global to avoid unnecessary syscalls
561561
static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);
562562

563+
let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
564+
NOT_PROBED => false,
565+
UNAVAILABLE => return CopyResult::Fallback(0),
566+
_ => true,
567+
};
568+
563569
syscall! {
564570
fn copy_file_range(
565571
fd_in: libc::c_int,
@@ -571,26 +577,6 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
571577
) -> libc::ssize_t
572578
}
573579

574-
match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
575-
NOT_PROBED => {
576-
// EPERM can indicate seccomp filters or an immutable file.
577-
// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported
578-
// and some other error (ENOSYS or EPERM) if it's not available
579-
let result = unsafe {
580-
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
581-
};
582-
583-
if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {
584-
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
585-
} else {
586-
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
587-
return CopyResult::Fallback(0);
588-
}
589-
}
590-
UNAVAILABLE => return CopyResult::Fallback(0),
591-
_ => {}
592-
};
593-
594580
let mut written = 0u64;
595581
while written < max_len {
596582
let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
@@ -604,6 +590,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
604590
cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
605591
};
606592

593+
if !have_probed && copy_result.is_ok() {
594+
have_probed = true;
595+
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
596+
}
597+
607598
match copy_result {
608599
Ok(0) if written == 0 => {
609600
// fallback to work around several kernel bugs where copy_file_range will fail to
@@ -616,10 +607,44 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
616607
Ok(0) => return CopyResult::Ended(written), // reached EOF
617608
Ok(ret) => written += ret as u64,
618609
Err(err) => {
619-
return match err.raw_os_error() {
610+
let raw_os_error = match err.raw_os_error() {
611+
Some(raw) => raw,
612+
_ => return CopyResult::Error(err, written),
613+
};
614+
return match raw_os_error {
620615
// when file offset + max_length > u64::MAX
621-
Some(EOVERFLOW) => CopyResult::Fallback(written),
622-
Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {
616+
EOVERFLOW => CopyResult::Fallback(written),
617+
ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF if written == 0 => {
618+
if !have_probed {
619+
if raw_os_error == ENOSYS {
620+
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
621+
} else {
622+
// EPERM can indicate seccomp filters or an
623+
// immutable file. To distinguish these cases
624+
// we probe with invalid file descriptors which
625+
// should result in EBADF if the syscall is
626+
// supported and some other error (ENOSYS or
627+
// EPERM) if it's not available.
628+
let result = unsafe {
629+
cvt(copy_file_range(
630+
INVALID_FD,
631+
ptr::null_mut(),
632+
INVALID_FD,
633+
ptr::null_mut(),
634+
1,
635+
0,
636+
))
637+
};
638+
639+
if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF)))
640+
{
641+
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
642+
} else {
643+
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
644+
}
645+
}
646+
}
647+
623648
// Try fallback io::copy if either:
624649
// - Kernel version is < 4.5 (ENOSYS¹)
625650
// - Files are mounted on different fs (EXDEV)

0 commit comments

Comments
 (0)
Failed to load comments.