1use crate::{BLOCK_SIZE, Extents, File};
6use anyhow::{Error, anyhow, bail};
7use fuchsia_sync::Mutex;
8use std::cmp::{Ordering, Reverse, max, min};
9use std::collections::BinaryHeap;
10use std::ops::{ControlFlow, Range};
11use std::sync::Arc;
12
13pub const MAX_READ_BUFFER_SIZE: usize = 1024 * 1024;
15
16pub use storage_device::SplittableBuffer;
17pub use storage_device::buffer::OwnedBuffer;
18
19pub trait BlockService: Send + Sync {
21 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer;
27
28 fn read_blocks(
31 &self,
32 device_offset: u64,
33 dest_buffer: OwnedBuffer,
34 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
35 ) -> Result<(), Error>;
36}
37
38struct BufferedChunk {
39 index: usize,
40 buffer: OwnedBuffer,
41}
42
43impl PartialEq for BufferedChunk {
44 fn eq(&self, other: &Self) -> bool {
45 self.index == other.index
46 }
47}
48
49impl Eq for BufferedChunk {}
50
51impl PartialOrd for BufferedChunk {
52 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
53 Some(self.cmp(other))
54 }
55}
56
57impl Ord for BufferedChunk {
58 fn cmp(&self, other: &Self) -> Ordering {
59 self.index.cmp(&other.index)
60 }
61}
62
63struct ReadContext<F>
68where
69 F: FnMut(Result<OwnedBuffer, Error>) -> ControlFlow<()> + Send + 'static,
70{
71 callback: Option<F>,
72
73 next_expected_chunk: usize,
75
76 buffered_chunks: BinaryHeap<Reverse<BufferedChunk>>,
78
79 total_chunks: Option<usize>,
81}
82
83impl<F> ReadContext<F>
84where
85 F: FnMut(Result<OwnedBuffer, Error>) -> ControlFlow<()> + Send + 'static,
86{
87 fn on_block_read_completed(
92 context: &Mutex<ReadContext<F>>,
93 chunk_index: usize,
94 res: Result<OwnedBuffer, Error>,
95 ) {
96 let mut guard = context.lock();
97 if guard.callback.is_none() {
98 return;
99 }
100
101 let mut current_buf = match res {
102 Ok(buf) => buf,
103 Err(e) => {
104 guard.buffered_chunks.clear();
105 let mut cb = guard.callback.take().unwrap();
106 drop(guard);
107 let _ = cb(Err(e));
108 return;
109 }
110 };
111
112 if chunk_index != guard.next_expected_chunk {
113 guard
114 .buffered_chunks
115 .push(Reverse(BufferedChunk { index: chunk_index, buffer: current_buf }));
116 return;
117 }
118
119 loop {
120 guard.next_expected_chunk += 1;
121 let cb = guard.callback.as_mut().unwrap();
122 if cb(Ok(current_buf)).is_break() {
123 guard.buffered_chunks.clear();
124 guard.callback = None;
125 return;
126 }
127
128 if let Some(top) = guard.buffered_chunks.peek()
129 && top.0.index == guard.next_expected_chunk
130 {
131 current_buf = guard.buffered_chunks.pop().unwrap().0.buffer;
132 } else {
133 break;
134 }
135 }
136 }
137}
138
139impl<F> Drop for ReadContext<F>
140where
141 F: FnMut(Result<OwnedBuffer, Error>) -> ControlFlow<()> + Send + 'static,
142{
143 fn drop(&mut self) {
144 if let Some(mut callback) = self.callback.take() {
145 let incomplete = match self.total_chunks {
146 Some(total) => {
147 self.next_expected_chunk != total || !self.buffered_chunks.is_empty()
148 }
149 None => true,
150 };
151 if incomplete {
152 let _ = callback(Err(anyhow!("ReadContext dropped before completion")));
153 }
154 }
155 }
156}
157
158pub fn read_aligned_range<F>(
172 extents: &Extents,
173 range: Range<u64>,
174 service: &(impl BlockService + ?Sized),
175 mut callback: F,
176) where
177 F: FnMut(Result<OwnedBuffer, Error>) -> ControlFlow<()> + Send + 'static,
178{
179 let mut offset = range.start;
180 let end = range.end;
181 assert_eq!(
182 offset % BLOCK_SIZE,
183 0,
184 "read_aligned_range: range.start ({offset}) must be block aligned ({BLOCK_SIZE})"
185 );
186 assert_eq!(
187 end % BLOCK_SIZE,
188 0,
189 "read_aligned_range: range.end ({end}) must be block aligned ({BLOCK_SIZE})"
190 );
191
192 if range.is_empty() {
193 let _ = callback(Err(anyhow!(
194 "read_aligned_range: range {range:?} must have a non-zero length"
195 )));
196 return;
197 }
198
199 let context = Arc::new(Mutex::new(ReadContext {
200 callback: Some(callback),
201 next_expected_chunk: 0,
202 buffered_chunks: BinaryHeap::new(),
203 total_chunks: None,
204 }));
205
206 let mut next_chunk_index = 0usize;
207
208 while offset < end {
209 if context.lock().callback.is_none() {
210 break;
211 }
212
213 let total_len = min(end - offset, MAX_READ_BUFFER_SIZE as u64) as usize;
214 let buffer = service.allocate_buffer(total_len);
215 let actual_len = buffer.len();
216 assert!(actual_len > 0 && actual_len <= total_len && actual_len % BLOCK_SIZE as usize == 0);
217
218 let actual_end = offset + actual_len as u64;
219
220 let chunk_index = next_chunk_index;
221 next_chunk_index += 1;
222
223 let context_clone = context.clone();
224 if let Err(e) = buffer.split(
225 |splittable| {
226 for extent in extents.iter_extents(offset) {
227 if extent.logical_range().start >= actual_end {
228 break;
229 }
230 let slice_start = max(extent.logical_range().start, offset);
231 let slice_end = min(extent.logical_range().end, actual_end);
232 let len_in_buf = (slice_end - slice_start) as usize;
233
234 if let Some(dev_offset) = extent.device_offset() {
235 let extent_dev_offset =
236 dev_offset + (slice_start - extent.logical_range().start);
237 let (child_buf, sub_handle) = splittable.take_prefix(len_in_buf);
238 service.read_blocks(
239 extent_dev_offset,
240 child_buf,
241 Box::new(move |res| {
242 sub_handle.merge(|| {
243 let _buf = res?;
244 Ok(())
245 });
246 }),
247 )?;
248 } else {
249 splittable.fill_zeros(len_in_buf);
250 }
251 }
252
253 let remaining_len = splittable.remaining_range().len();
254 if remaining_len > 0 {
255 bail!(
256 "read_aligned_range: requested range extends {remaining_len} bytes \
257 beyond the end of extents mappings"
258 );
259 }
260
261 Ok(())
262 },
263 move |res| ReadContext::on_block_read_completed(&context_clone, chunk_index, res),
264 ) {
265 ReadContext::on_block_read_completed(&context, chunk_index, Err(e));
266 return;
267 }
268
269 offset = actual_end;
270 }
271
272 context.lock().total_chunks = Some(next_chunk_index);
273}
274
275pub fn read_buffer_from_extents(
281 extents: &Extents,
282 logical_offset: u64,
283 dest_buffer: OwnedBuffer,
284 service: &(impl BlockService + ?Sized),
285 on_complete: impl FnOnce(Result<OwnedBuffer, Error>) + Send + 'static,
286) -> Result<(), Error> {
287 let total_len = dest_buffer.len() as u64;
288 if total_len == 0 {
289 on_complete(Ok(dest_buffer));
290 return Ok(());
291 }
292
293 dest_buffer.split(
294 |splittable| {
295 let end_offset = logical_offset + total_len;
296
297 if let Some(extent) = extents.iter_extents(logical_offset).next() {
299 let logical = extent.logical_range();
300 if logical.start <= logical_offset && logical.end >= end_offset {
301 if let Some(dev_offset) = extent.device_offset() {
302 let extent_dev_offset = dev_offset + (logical_offset - logical.start);
303 let (child_buf, sub_handle) = splittable.take_prefix(total_len as usize);
304 return service.read_blocks(
305 extent_dev_offset,
306 child_buf,
307 Box::new(move |res| {
308 sub_handle.merge(|| {
309 let _buf = res?;
310 Ok(())
311 });
312 }),
313 );
314 } else {
315 splittable.fill_zeros(total_len as usize);
317 return Ok(());
318 }
319 }
320 }
321
322 let mut current_offset = logical_offset;
324 for extent in extents.iter_extents(logical_offset) {
325 if extent.logical_range().start >= end_offset {
326 break;
327 }
328 let slice_start = max(extent.logical_range().start, current_offset);
329 let slice_end = min(extent.logical_range().end, end_offset);
330 if slice_start >= slice_end {
331 continue;
332 }
333
334 if slice_start > current_offset {
335 let hole_len = (slice_start - current_offset) as usize;
336 splittable.fill_zeros(hole_len);
337 }
338
339 let len_in_buf = (slice_end - slice_start) as usize;
340 if let Some(dev_offset) = extent.device_offset() {
341 let extent_dev_offset =
342 dev_offset + (slice_start - extent.logical_range().start);
343 let (child_buf, sub_handle) = splittable.take_prefix(len_in_buf);
344 service.read_blocks(
345 extent_dev_offset,
346 child_buf,
347 Box::new(move |res| {
348 sub_handle.merge(|| {
349 let _buf = res?;
350 Ok(())
351 });
352 }),
353 )?;
354 } else {
355 splittable.fill_zeros(len_in_buf);
356 }
357 current_offset = slice_end;
358 }
359
360 if current_offset < end_offset {
361 let hole_len = (end_offset - current_offset) as usize;
362 splittable.fill_zeros(hole_len);
363 }
364
365 Ok(())
366 },
367 on_complete,
368 )
369}
370
371pub struct ChildBlockService<S: ?Sized> {
374 parent_service: Arc<S>,
375 file: Arc<File>,
376}
377
378impl<S: BlockService + ?Sized> ChildBlockService<S> {
379 pub fn new(parent_service: Arc<S>, file: Arc<File>) -> Self {
380 Self { parent_service, file }
381 }
382
383 pub fn file(&self) -> &Arc<File> {
384 &self.file
385 }
386}
387
388impl<S: BlockService + ?Sized> BlockService for ChildBlockService<S> {
389 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
390 self.parent_service.allocate_buffer(max_len)
391 }
392
393 fn read_blocks(
394 &self,
395 device_offset: u64,
396 dest_buffer: OwnedBuffer,
397 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
398 ) -> Result<(), Error> {
399 read_buffer_from_extents(
400 self.file.extents(),
401 device_offset,
402 dest_buffer,
403 self.parent_service.as_ref(),
404 on_complete,
405 )
406 }
407}
408
409#[cfg(test)]
410pub(crate) mod tests {
411 use super::*;
412 use crate::{Extent, Transform};
413 use std::cmp::min;
414 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
415 use std::thread;
416 use std::time::Duration;
417 use storage_device::buffer_allocator::{BufferAllocator, BufferSource};
418
419 pub(crate) struct FakeBlockService {
420 allocator: Arc<BufferAllocator>,
421 device_data: Mutex<Vec<u8>>,
422 cap: Option<usize>,
423 read_count: AtomicUsize,
424 }
425
426 impl FakeBlockService {
427 pub(crate) fn new(device_data: Vec<u8>) -> Self {
428 Self::new_with_cap(device_data, None)
429 }
430
431 pub(crate) fn new_with_cap(device_data: Vec<u8>, cap: Option<usize>) -> Self {
432 Self::new_with_pool_size(device_data, 1024 * 1024, cap)
433 }
434
435 pub(crate) fn new_with_pool_size(
436 device_data: Vec<u8>,
437 pool_size: usize,
438 cap: Option<usize>,
439 ) -> Self {
440 let source = BufferSource::new(pool_size);
441 let allocator = Arc::new(BufferAllocator::new(4096, source));
442 Self {
443 allocator,
444 device_data: Mutex::new(device_data),
445 cap,
446 read_count: AtomicUsize::new(0),
447 }
448 }
449
450 pub(crate) fn read_count(&self) -> usize {
451 self.read_count.load(Ordering::Relaxed)
452 }
453 }
454
455 impl BlockService for FakeBlockService {
456 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
457 assert!(max_len > 0, "FakeBlockService::allocate_buffer: max_len must be > 0");
458 let len = match self.cap {
459 Some(cap) => min(max_len, cap),
460 None => max_len,
461 };
462 self.allocator.allocate_buffer_sync_owned(len)
463 }
464
465 fn read_blocks(
466 &self,
467 device_offset: u64,
468 mut dest_buffer: OwnedBuffer,
469 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
470 ) -> Result<(), Error> {
471 self.read_count.fetch_add(1, Ordering::Relaxed);
472 assert_eq!(
473 device_offset % 512,
474 0,
475 "FakeBlockService::read_blocks: device_offset ({device_offset}) must be 512-byte \
476 aligned"
477 );
478 let len = dest_buffer.len();
479 assert_eq!(
480 len % BLOCK_SIZE as usize,
481 0,
482 "FakeBlockService::read_blocks: dest_buffer.len() ({len}) must be block aligned"
483 );
484 let start = device_offset as usize;
485 let end = start + dest_buffer.len();
486 let data = self.device_data.lock();
487 dest_buffer.copy_from_slice(&data[start..end]);
488 on_complete(Ok(dest_buffer));
489 Ok(())
490 }
491 }
492
493 #[test]
494 #[should_panic(expected = "must be block aligned")]
495 fn test_read_aligned_range_unaligned_panics() {
496 let service = Arc::new(FakeBlockService::new(vec![0u8; 8192]));
497 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
498
499 read_aligned_range(&mappings, 1000..5000, &*service, |_| ControlFlow::Continue(()));
500 }
501
502 #[test]
503 #[allow(clippy::reversed_empty_ranges)]
504 fn test_read_aligned_range_empty_range_errors() {
505 let service = FakeBlockService::new(vec![0u8; 8192]);
506 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
507
508 let err_count = Arc::new(AtomicUsize::new(0));
509 let err_count_clone = err_count.clone();
510 read_aligned_range(&mappings, 4096..4096, &service, move |res| {
511 assert!(res.is_err());
512 err_count_clone.fetch_add(1, Ordering::Relaxed);
513 ControlFlow::Continue(())
514 });
515 assert_eq!(err_count.load(Ordering::Relaxed), 1);
516
517 let err_count_clone2 = err_count.clone();
518 read_aligned_range(&mappings, 8192..4096, &service, move |res| {
519 assert!(res.is_err());
520 err_count_clone2.fetch_add(1, Ordering::Relaxed);
521 ControlFlow::Continue(())
522 });
523 assert_eq!(err_count.load(Ordering::Relaxed), 2);
524 }
525
526 #[test]
527 fn test_read_aligned_range_beyond_extents_errors() {
528 let service = FakeBlockService::new(vec![0u8; 16384]);
529 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
531
532 let err_occurred = Arc::new(AtomicBool::new(false));
533 let err_occurred_clone = err_occurred.clone();
534 read_aligned_range(&mappings, 0..16384, &service, move |res| {
535 if res.is_err() {
536 err_occurred_clone.store(true, Ordering::Relaxed);
537 }
538 ControlFlow::Continue(())
539 });
540 assert!(err_occurred.load(Ordering::Relaxed));
541 }
542
543 #[test]
544 fn test_read_aligned_range_single_extent() {
545 let mut device_data = vec![0u8; 16384];
546 device_data[4096..8192].copy_from_slice(&[42u8; 4096]);
547 let service = Arc::new(FakeBlockService::new(device_data));
548
549 let mappings = Extents::try_new([Extent::new(0..4096, Some(4096))], 0).unwrap();
550
551 let completed = Arc::new(AtomicBool::new(false));
552 let completed_clone = completed.clone();
553
554 read_aligned_range(&mappings, 0..4096, &*service, move |res| {
555 let buffer = res.expect("read_aligned_range should succeed");
556 assert_eq!(buffer.len(), 4096);
557 assert!(buffer.as_ptr_slice().iter_as::<u8>().all(|b| b == 42));
558 completed_clone.store(true, Ordering::Relaxed);
559 ControlFlow::Continue(())
560 });
561
562 assert!(completed.load(Ordering::Relaxed));
563 }
564
565 #[test]
566 fn test_read_aligned_range_multi_extent() {
567 let mut device_data = vec![0u8; 16384];
568 device_data[0..4096].fill(1);
569 device_data[8192..12288].fill(2);
570 let service = Arc::new(FakeBlockService::new(device_data));
571
572 let mappings = Extents::try_new(
573 [Extent::new(0..4096, Some(0)), Extent::new(4096..8192, Some(8192))],
574 0,
575 )
576 .unwrap();
577
578 let completed = Arc::new(AtomicBool::new(false));
579 let completed_clone = completed.clone();
580 let mut received_bytes = Vec::new();
581
582 read_aligned_range(&mappings, 0..8192, &*service, move |res| {
583 let buffer = res.expect("read_aligned_range should succeed");
584 buffer.as_ptr_slice().append_to(&mut received_bytes);
585 if received_bytes.len() == 8192 {
586 assert!(received_bytes[0..4096].iter().all(|&b| b == 1));
587 assert!(received_bytes[4096..8192].iter().all(|&b| b == 2));
588 completed_clone.store(true, Ordering::Relaxed);
589 }
590 ControlFlow::Continue(())
591 });
592
593 assert!(completed.load(Ordering::Relaxed));
594 }
595
596 #[test]
597 fn test_read_aligned_range_sparse_extent() {
598 let service = Arc::new(FakeBlockService::new(vec![0u8; 4096]));
599
600 let mappings = Extents::try_new([Extent::new(0..4096, None)], 0).unwrap();
601
602 let completed = Arc::new(AtomicBool::new(false));
603 let completed_clone = completed.clone();
604
605 read_aligned_range(&mappings, 0..4096, &*service, move |res| {
606 let buffer = res.expect("read_aligned_range should succeed");
607 assert_eq!(buffer.len(), 4096);
608 assert!(buffer.as_ptr_slice().iter_as::<u64>().all(|b| b == 0));
609 completed_clone.store(true, Ordering::Relaxed);
610 ControlFlow::Continue(())
611 });
612
613 assert!(completed.load(Ordering::Relaxed));
614 }
615
616 #[test]
617 fn test_read_aligned_range_partial_buffer_chaining() {
618 let mut device_data = vec![0u8; 16384];
619 device_data[0..4096].fill(1);
620 device_data[4096..8192].fill(2);
621 let service = Arc::new(FakeBlockService::new_with_cap(device_data, Some(4096)));
623
624 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
625
626 let completed = Arc::new(AtomicBool::new(false));
627 let completed_clone = completed.clone();
628 let mut chunk_count = 0;
629 let mut received_bytes = Vec::new();
630
631 read_aligned_range(&mappings, 0..8192, &*service, move |res| {
632 let buffer = res.expect("read_aligned_range should succeed");
633 chunk_count += 1;
634 buffer.as_ptr_slice().append_to(&mut received_bytes);
635 if received_bytes.len() == 8192 {
636 assert_eq!(chunk_count, 2, "Should have streamed in 2 rounds of 4096 bytes");
637 assert!(received_bytes[0..4096].iter().all(|&b| b == 1));
638 assert!(received_bytes[4096..8192].iter().all(|&b| b == 2));
639 completed_clone.store(true, Ordering::Relaxed);
640 }
641 ControlFlow::Continue(())
642 });
643
644 assert!(completed.load(Ordering::Relaxed));
645 }
646
647 struct OutOfOrderBlockService {
648 inner: FakeBlockService,
649 delayed: Mutex<Vec<Box<dyn FnOnce() + Send>>>,
650 }
651
652 impl BlockService for OutOfOrderBlockService {
653 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
654 self.inner.allocate_buffer(max_len)
655 }
656
657 fn read_blocks(
658 &self,
659 device_offset: u64,
660 dest_buffer: OwnedBuffer,
661 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
662 ) -> Result<(), Error> {
663 if device_offset == 0 {
664 let inner = &self.inner;
666 let data = inner.device_data.lock();
667 let start = device_offset as usize;
668 let end = start + dest_buffer.len();
669 let mut buf = dest_buffer;
670 buf.copy_from_slice(&data[start..end]);
671 self.delayed.lock().push(Box::new(move || on_complete(Ok(buf))));
672 Ok(())
673 } else if device_offset == 4096 {
674 self.inner.read_blocks(device_offset, dest_buffer, on_complete)?;
676 Ok(())
677 } else {
678 self.inner.read_blocks(device_offset, dest_buffer, on_complete)?;
682 let mut delayed = self.delayed.lock();
683 for cb in delayed.drain(..) {
684 cb();
685 }
686 Ok(())
687 }
688 }
689 }
690
691 #[test]
692 fn test_read_aligned_range_out_of_order_reordering() {
693 let mut device_data = vec![0u8; 16384];
694 device_data[0..4096].fill(10);
695 device_data[4096..8192].fill(20);
696 device_data[8192..12288].fill(30);
697 let inner = FakeBlockService::new_with_cap(device_data, Some(4096));
698 let service = Arc::new(OutOfOrderBlockService { inner, delayed: Mutex::new(Vec::new()) });
699
700 let mappings = Extents::try_new(
701 [
702 Extent::new(0..4096, Some(0)),
703 Extent::new(4096..8192, Some(4096)),
704 Extent::new(8192..12288, Some(8192)),
705 ],
706 0,
707 )
708 .unwrap();
709
710 let completed = Arc::new(AtomicBool::new(false));
711 let completed_clone = completed.clone();
712 let mut received_chunks = Vec::new();
713
714 read_aligned_range(&mappings, 0..12288, &*service, move |res| {
715 let buffer = res.expect("read_aligned_range should succeed");
716 let first_byte = buffer.as_ptr_slice().to_vec()[0];
717 received_chunks.push(first_byte);
718 if received_chunks.len() == 3 {
719 assert_eq!(received_chunks, vec![10, 20, 30]);
721 completed_clone.store(true, Ordering::Relaxed);
722 }
723 ControlFlow::Continue(())
724 });
725
726 assert!(completed.load(Ordering::Relaxed));
727 }
728
729 struct ThreadedBlockService {
730 inner: FakeBlockService,
731 }
732
733 impl BlockService for ThreadedBlockService {
734 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
735 self.inner.allocate_buffer(max_len)
736 }
737
738 fn read_blocks(
739 &self,
740 device_offset: u64,
741 mut dest_buffer: OwnedBuffer,
742 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
743 ) -> Result<(), Error> {
744 let start = device_offset as usize;
745 let end = start + dest_buffer.len();
746 let data = self.inner.device_data.lock();
747 dest_buffer.copy_from_slice(&data[start..end]);
748 drop(data);
749
750 thread::spawn(move || {
753 thread::sleep(Duration::from_millis(15));
754 on_complete(Ok(dest_buffer));
755 });
756 Ok(())
757 }
758 }
759
760 #[test]
761 fn test_read_aligned_range_limited_memory_blocking_allocator() {
762 let mut device_data = vec![0u8; 16384];
763 device_data[0..4096].fill(1);
764 device_data[4096..8192].fill(2);
765 device_data[8192..12288].fill(3);
766 device_data[12288..16384].fill(4);
767
768 let inner = FakeBlockService::new_with_pool_size(device_data, 8192, Some(4096));
774 let service = Arc::new(ThreadedBlockService { inner });
775
776 let mappings = Extents::try_new([Extent::new(0..16384, Some(0))], 0).unwrap();
777
778 let completed = Arc::new(AtomicBool::new(false));
779 let completed_clone = completed.clone();
780 let mut received_bytes = Vec::new();
781
782 read_aligned_range(&mappings, 0..16384, &*service, move |res| {
783 let buffer = res.expect("read_aligned_range should succeed");
784 buffer.as_ptr_slice().append_to(&mut received_bytes);
785 if received_bytes.len() == 16384 {
786 assert!(received_bytes[0..4096].iter().all(|&b| b == 1));
787 assert!(received_bytes[4096..8192].iter().all(|&b| b == 2));
788 assert!(received_bytes[8192..12288].iter().all(|&b| b == 3));
789 assert!(received_bytes[12288..16384].iter().all(|&b| b == 4));
790 completed_clone.store(true, Ordering::Relaxed);
791 }
792 ControlFlow::Continue(())
793 });
794
795 for _ in 0..100 {
797 if completed.load(Ordering::Relaxed) {
798 break;
799 }
800 thread::sleep(Duration::from_millis(10));
801 }
802 assert!(completed.load(Ordering::Relaxed));
803 }
804
805 #[test]
806 fn test_read_aligned_range_max_read_buffer_size_capping() {
807 struct CapturingBlockService {
808 inner: FakeBlockService,
809 requested_lens: Mutex<Vec<usize>>,
810 }
811
812 impl BlockService for CapturingBlockService {
813 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
814 self.requested_lens.lock().push(max_len);
815 self.inner.allocate_buffer(max_len)
816 }
817
818 fn read_blocks(
819 &self,
820 device_offset: u64,
821 dest_buffer: OwnedBuffer,
822 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
823 ) -> Result<(), Error> {
824 self.inner.read_blocks(device_offset, dest_buffer, on_complete)
825 }
826 }
827
828 let inner = FakeBlockService::new(vec![0u8; MAX_READ_BUFFER_SIZE + 8192]);
829 let service = CapturingBlockService { inner, requested_lens: Mutex::new(Vec::new()) };
830 let mappings =
831 Extents::try_new([Extent::new(0..(MAX_READ_BUFFER_SIZE + 8192) as u64, Some(0))], 0)
832 .unwrap();
833
834 read_aligned_range(&mappings, 0..(MAX_READ_BUFFER_SIZE + 8192) as u64, &service, |_| {
835 ControlFlow::Continue(())
836 });
837
838 assert_eq!(*service.requested_lens.lock(), vec![MAX_READ_BUFFER_SIZE, 8192]);
839 }
840
841 #[test]
842 fn test_read_aligned_range_sync_read_blocks_error() {
843 struct SyncErrorBlockService(FakeBlockService);
844 impl BlockService for SyncErrorBlockService {
845 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
846 self.0.allocate_buffer(max_len)
847 }
848 fn read_blocks(
849 &self,
850 _offset: u64,
851 _dest: OwnedBuffer,
852 _on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
853 ) -> Result<(), Error> {
854 Err(anyhow!("synchronous read_blocks failure"))
855 }
856 }
857 let service = SyncErrorBlockService(FakeBlockService::new(vec![0u8; 4096]));
858 let mappings = Extents::try_new([Extent::new(0..4096, Some(0))], 0).unwrap();
859
860 let err_received = Arc::new(AtomicBool::new(false));
861 let err_clone = err_received.clone();
862 read_aligned_range(&mappings, 0..4096, &service, move |res| {
863 assert!(res.is_err());
864 err_clone.store(true, Ordering::Relaxed);
865 ControlFlow::Continue(())
866 });
867 assert!(err_received.load(Ordering::Relaxed));
868 }
869
870 #[test]
871 fn test_read_aligned_range_async_callback_errors() {
872 struct AsyncErrorBlockService(FakeBlockService, u64);
873 impl BlockService for AsyncErrorBlockService {
874 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
875 self.0.allocate_buffer(max_len)
876 }
877 fn read_blocks(
878 &self,
879 device_offset: u64,
880 dest_buffer: OwnedBuffer,
881 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
882 ) -> Result<(), Error> {
883 if device_offset == self.1 {
884 on_complete(Err(anyhow!("async block read error on offset {device_offset}")));
885 Ok(())
886 } else {
887 self.0.read_blocks(device_offset, dest_buffer, on_complete)
888 }
889 }
890 }
891
892 let service = AsyncErrorBlockService(FakeBlockService::new(vec![0u8; 8192]), 0);
894 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
895 let err_count = Arc::new(AtomicUsize::new(0));
896 let err_clone = err_count.clone();
897 read_aligned_range(&mappings, 0..8192, &service, move |res| {
898 if res.is_err() {
899 err_clone.fetch_add(1, Ordering::Relaxed);
900 }
901 ControlFlow::Continue(())
902 });
903 assert_eq!(err_count.load(Ordering::Relaxed), 1);
904
905 let service = Arc::new(OutOfOrderBlockService {
907 inner: FakeBlockService::new(vec![0u8; 8192]),
908 delayed: Mutex::new(Vec::new()),
909 });
910 struct FailingDelayedService(Arc<OutOfOrderBlockService>);
912 impl BlockService for FailingDelayedService {
913 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
914 self.0.allocate_buffer(max_len)
915 }
916 fn read_blocks(
917 &self,
918 device_offset: u64,
919 dest_buffer: OwnedBuffer,
920 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
921 ) -> Result<(), Error> {
922 if device_offset == 0 {
923 self.0.delayed.lock().push(Box::new(move || {
924 on_complete(Err(anyhow!("delayed chunk 0 failure")))
925 }));
926 Ok(())
927 } else {
928 let res = self.0.read_blocks(device_offset, dest_buffer, on_complete);
929 let mut delayed = self.0.delayed.lock();
930 for cb in delayed.drain(..) {
931 cb();
932 }
933 res
934 }
935 }
936 }
937 let failing_service = FailingDelayedService(service.clone());
938 let err_count = Arc::new(AtomicUsize::new(0));
939 let err_clone = err_count.clone();
940 read_aligned_range(&mappings, 0..8192, &failing_service, move |res| {
941 if res.is_err() {
942 err_clone.fetch_add(1, Ordering::Relaxed);
943 }
944 ControlFlow::Continue(())
945 });
946 let mut delayed = service.delayed.lock();
947 for cb in delayed.drain(..) {
948 cb();
949 }
950 assert_eq!(err_count.load(Ordering::Relaxed), 1);
951 }
952
953 #[test]
954 fn test_read_context_dropped_before_completion() {
955 struct DroppingBlockService(Mutex<Vec<Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>>>);
956 impl BlockService for DroppingBlockService {
957 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
958 FakeBlockService::new(vec![0u8; max_len]).allocate_buffer(max_len)
959 }
960 fn read_blocks(
961 &self,
962 _offset: u64,
963 _dest: OwnedBuffer,
964 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
965 ) -> Result<(), Error> {
966 self.0.lock().push(on_complete);
967 Ok(())
968 }
969 }
970 let service = DroppingBlockService(Mutex::new(Vec::new()));
971 let mappings = Extents::try_new([Extent::new(0..4096, Some(0))], 0).unwrap();
972
973 let err_msg = Arc::new(Mutex::new(String::new()));
974 let err_msg_clone = err_msg.clone();
975 read_aligned_range(&mappings, 0..4096, &service, move |res| {
976 if let Err(e) = res {
977 *err_msg_clone.lock() = e.to_string();
978 }
979 ControlFlow::Continue(())
980 });
981 service.0.lock().clear();
982 assert_eq!(*err_msg.lock(), "Read sub-request dropped before completion");
983 }
984
985 #[test]
986 fn test_read_aligned_range_multi_iteration_error_break() {
987 struct FirstIterSyncErrorService(FakeBlockService);
988 impl BlockService for FirstIterSyncErrorService {
989 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
990 self.0.allocate_buffer(max_len)
991 }
992 fn read_blocks(
993 &self,
994 device_offset: u64,
995 dest_buffer: OwnedBuffer,
996 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
997 ) -> Result<(), Error> {
998 if device_offset == 0 {
999 Err(anyhow!("sync failure on first iteration"))
1000 } else {
1001 self.0.read_blocks(device_offset, dest_buffer, on_complete)
1002 }
1003 }
1004 }
1005 let inner = FakeBlockService::new_with_cap(vec![0u8; 8192], Some(4096));
1006 let service = FirstIterSyncErrorService(inner);
1007 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
1008
1009 let err_received = Arc::new(AtomicBool::new(false));
1010 let err_clone = err_received.clone();
1011 read_aligned_range(&mappings, 0..8192, &service, move |res| {
1012 if res.is_err() {
1013 err_clone.store(true, Ordering::Relaxed);
1014 }
1015 ControlFlow::Continue(())
1016 });
1017 assert!(err_received.load(Ordering::Relaxed));
1018 }
1019
1020 #[test]
1021 fn test_read_aligned_range_extent_beyond_actual_end_break() {
1022 let inner = FakeBlockService::new_with_cap(vec![0u8; 8192], Some(4096));
1023 let mappings = Extents::try_new(
1024 [Extent::new(0..4096, Some(0)), Extent::new(4096..8192, Some(4096))],
1025 0,
1026 )
1027 .unwrap();
1028
1029 let count = Arc::new(AtomicUsize::new(0));
1030 let count_clone = count.clone();
1031 read_aligned_range(&mappings, 0..8192, &inner, move |res| {
1032 if res.is_ok() {
1033 count_clone.fetch_add(1, Ordering::Relaxed);
1034 }
1035 ControlFlow::Continue(())
1036 });
1037 assert_eq!(count.load(Ordering::Relaxed), 2);
1038 }
1039
1040 #[test]
1041 fn test_read_aligned_range_merged_extents() {
1042 let mut device_data = vec![0u8; 32768];
1043 device_data[4096..8192].fill(0xAA);
1044 device_data[8192..20480].fill(0xBB);
1045 let service = Arc::new(FakeBlockService::new(device_data));
1046
1047 let mappings = Extents::try_new(
1048 [Extent::new(0..4096, Some(4096)), Extent::new(4096..16384, Some(8192))],
1049 0,
1050 )
1051 .unwrap();
1052
1053 let count = Arc::new(AtomicUsize::new(0));
1054 let count_clone = count.clone();
1055 let completed = Arc::new(AtomicBool::new(false));
1056 let completed_clone = completed.clone();
1057
1058 read_aligned_range(&mappings, 0..16384, &*service, move |res| {
1059 count_clone.fetch_add(1, Ordering::Relaxed);
1060 let buffer = res.expect("merged read should succeed");
1061 assert_eq!(buffer.len(), 16384);
1062 assert!(buffer.as_ptr_slice().subslice(0..4096).iter_as::<u8>().all(|b| b == 0xAA));
1063 assert!(buffer.as_ptr_slice().subslice(4096..16384).iter_as::<u8>().all(|b| b == 0xBB));
1064 completed_clone.store(true, Ordering::Relaxed);
1065 ControlFlow::Continue(())
1066 });
1067
1068 assert_eq!(count.load(Ordering::Relaxed), 1);
1069 assert!(completed.load(Ordering::Relaxed));
1070 }
1071
1072 #[test]
1073 fn test_read_aligned_range_callback_error_propagation() {
1074 let service = Arc::new(FakeBlockService::new(vec![0u8; 8192]));
1075 let mappings = Extents::try_new([Extent::new(0..8192, Some(0))], 0).unwrap();
1076
1077 let call_count = Arc::new(AtomicUsize::new(0));
1078 let call_count_clone = call_count.clone();
1079 read_aligned_range(&mappings, 0..8192, &*service, move |_res| {
1080 call_count_clone.fetch_add(1, Ordering::Relaxed);
1081 ControlFlow::Break(())
1082 });
1083
1084 assert_eq!(call_count.load(Ordering::Relaxed), 1);
1085 }
1086
1087 #[test]
1088 fn test_read_buffer_from_extents_single_extent() {
1089 let mut device_data = vec![0u8; 16384];
1090 device_data[4096..8192].fill(0xAA);
1091 let service = Arc::new(FakeBlockService::new(device_data));
1092
1093 let mappings = Extents::try_new([Extent::new(0..4096, Some(4096))], 0).unwrap();
1095
1096 let dest = service.allocate_buffer(4096);
1097 let (tx, rx) = std::sync::mpsc::channel();
1098 read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |res| {
1099 tx.send(res).unwrap();
1100 })
1101 .unwrap();
1102
1103 let buffer = rx.recv().unwrap().unwrap();
1104 assert_eq!(buffer.len(), 4096);
1105 assert!(buffer.as_ptr_slice().iter_as::<u8>().all(|b| b == 0xAA));
1106 }
1107
1108 #[test]
1109 fn test_read_buffer_from_extents_sparse_hole_fast_path() {
1110 let service = Arc::new(FakeBlockService::new(vec![0u8; 16384]));
1111
1112 let mappings = Extents::try_new([Extent::new(0..4096, None)], 0).unwrap();
1114
1115 let dest = service.allocate_buffer(4096);
1116 let (tx, rx) = std::sync::mpsc::channel();
1117 read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |res| {
1118 tx.send(res).unwrap();
1119 })
1120 .unwrap();
1121
1122 let buffer = rx.recv().unwrap().unwrap();
1123 assert_eq!(buffer.len(), 4096);
1124 assert!(buffer.as_ptr_slice().iter_as::<u8>().all(|b| b == 0x00));
1125 }
1126
1127 #[test]
1128 fn test_read_buffer_from_extents_multi_extent_and_holes() {
1129 let mut device_data = vec![0u8; 32768];
1130 device_data[0..4096].fill(0x11);
1131 device_data[8192..12288].fill(0x22);
1132 let service = Arc::new(FakeBlockService::new(device_data));
1133
1134 let mappings = Extents::try_new(
1139 [
1140 Extent::new(0..4096, Some(0)),
1141 Extent::new(4096..8192, None),
1142 Extent::new(8192..12288, Some(8192)),
1143 ],
1144 0,
1145 )
1146 .unwrap();
1147
1148 let dest = service.allocate_buffer(12288);
1149 let (tx, rx) = std::sync::mpsc::channel();
1150 read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |res| {
1151 tx.send(res).unwrap();
1152 })
1153 .unwrap();
1154
1155 let buffer = rx.recv().unwrap().unwrap();
1156 assert_eq!(buffer.len(), 12288);
1157 assert!(buffer.as_ptr_slice().subslice(0..4096).iter_as::<u8>().all(|b| b == 0x11));
1158 assert!(buffer.as_ptr_slice().subslice(4096..8192).iter_as::<u8>().all(|b| b == 0x00));
1159 assert!(buffer.as_ptr_slice().subslice(8192..12288).iter_as::<u8>().all(|b| b == 0x22));
1160 }
1161
1162 #[test]
1163 fn test_read_buffer_from_extents_sync_error_disarms_callback() {
1164 struct SecondChunkSyncErrorService {
1165 inner: FakeBlockService,
1166 }
1167 impl BlockService for SecondChunkSyncErrorService {
1168 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
1169 self.inner.allocate_buffer(max_len)
1170 }
1171 fn read_blocks(
1172 &self,
1173 device_offset: u64,
1174 dest_buffer: OwnedBuffer,
1175 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
1176 ) -> Result<(), Error> {
1177 if device_offset == 8192 {
1178 Err(anyhow!("sync failure on second chunk"))
1179 } else {
1180 self.inner.read_blocks(device_offset, dest_buffer, on_complete)
1181 }
1182 }
1183 }
1184
1185 let inner = FakeBlockService::new(vec![0u8; 32768]);
1186 let service = Arc::new(SecondChunkSyncErrorService { inner });
1187
1188 let mappings = Extents::try_new(
1190 [Extent::new(0..4096, Some(0)), Extent::new(4096..8192, Some(8192))],
1191 0,
1192 )
1193 .unwrap();
1194
1195 let dest = service.allocate_buffer(8192);
1196 let completed = Arc::new(AtomicBool::new(false));
1197 let completed_clone = completed.clone();
1198
1199 let res = read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |_res| {
1200 completed_clone.store(true, Ordering::Relaxed);
1201 });
1202
1203 assert!(res.is_err());
1205 assert!(!completed.load(Ordering::Relaxed));
1208 }
1209
1210 #[test]
1211 fn test_read_buffer_from_extents_in_flight_chunk_succeeds_after_sync_error() {
1212 struct InFlightFirstChunkService {
1213 inner: FakeBlockService,
1214 pending_first_chunk:
1215 Mutex<Option<(OwnedBuffer, Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>)>>,
1216 }
1217 impl BlockService for InFlightFirstChunkService {
1218 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
1219 self.inner.allocate_buffer(max_len)
1220 }
1221 fn read_blocks(
1222 &self,
1223 device_offset: u64,
1224 dest_buffer: OwnedBuffer,
1225 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
1226 ) -> Result<(), Error> {
1227 if device_offset == 0 {
1228 *self.pending_first_chunk.lock() = Some((dest_buffer, on_complete));
1230 Ok(())
1231 } else if device_offset == 8192 {
1232 Err(anyhow!("sync failure on second chunk"))
1234 } else {
1235 self.inner.read_blocks(device_offset, dest_buffer, on_complete)
1236 }
1237 }
1238 }
1239
1240 let inner = FakeBlockService::new(vec![0u8; 32768]);
1241 let service =
1242 Arc::new(InFlightFirstChunkService { inner, pending_first_chunk: Mutex::new(None) });
1243
1244 let mappings = Extents::try_new(
1246 [Extent::new(0..4096, Some(0)), Extent::new(4096..8192, Some(8192))],
1247 0,
1248 )
1249 .unwrap();
1250
1251 let dest = service.allocate_buffer(8192);
1252 let completed = Arc::new(AtomicBool::new(false));
1253 let completed_clone = completed.clone();
1254
1255 let res = read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |_res| {
1256 completed_clone.store(true, Ordering::Relaxed);
1257 });
1258
1259 assert!(res.is_err());
1261 assert!(!completed.load(Ordering::Relaxed));
1262
1263 let (buf, cb) = service.pending_first_chunk.lock().take().expect("chunk 1 pending");
1265 cb(Ok(buf));
1266
1267 assert!(!completed.load(Ordering::Relaxed));
1269 }
1270
1271 #[test]
1272 fn test_read_buffer_from_extents_async_error_disarms_subsequent_chunks() {
1273 struct AsyncErrorService {
1274 inner: FakeBlockService,
1275 pending_chunks:
1276 Mutex<Vec<(OwnedBuffer, Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>, u64)>>,
1277 }
1278 impl BlockService for AsyncErrorService {
1279 fn allocate_buffer(&self, max_len: usize) -> OwnedBuffer {
1280 self.inner.allocate_buffer(max_len)
1281 }
1282 fn read_blocks(
1283 &self,
1284 device_offset: u64,
1285 dest_buffer: OwnedBuffer,
1286 on_complete: Box<dyn FnOnce(Result<OwnedBuffer, Error>) + Send>,
1287 ) -> Result<(), Error> {
1288 self.pending_chunks.lock().push((dest_buffer, on_complete, device_offset));
1289 Ok(())
1290 }
1291 }
1292
1293 let inner = FakeBlockService::new(vec![0u8; 32768]);
1294 let service = Arc::new(AsyncErrorService { inner, pending_chunks: Mutex::new(Vec::new()) });
1295
1296 let mappings = Extents::try_new(
1297 [Extent::new(0..4096, Some(0)), Extent::new(4096..8192, Some(8192))],
1298 0,
1299 )
1300 .unwrap();
1301
1302 let dest = service.allocate_buffer(8192);
1303 let err_count = Arc::new(AtomicUsize::new(0));
1304 let ok_count = Arc::new(AtomicUsize::new(0));
1305 let err_count_clone = err_count.clone();
1306 let ok_count_clone = ok_count.clone();
1307
1308 let res = read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |res| {
1309 if res.is_ok() {
1310 ok_count_clone.fetch_add(1, Ordering::Relaxed);
1311 } else {
1312 err_count_clone.fetch_add(1, Ordering::Relaxed);
1313 }
1314 });
1315 assert!(res.is_ok());
1316
1317 let mut chunks = service.pending_chunks.lock();
1318 assert_eq!(chunks.len(), 2);
1319 let (buf1, cb1, _) = chunks.remove(0);
1320 let (buf2, cb2, _) = chunks.remove(0);
1321 drop(chunks);
1322
1323 drop(buf1);
1325 cb1(Err(anyhow!("async failure on chunk 1")));
1326 assert_eq!(err_count.load(Ordering::Relaxed), 1);
1327 assert_eq!(ok_count.load(Ordering::Relaxed), 0);
1328
1329 cb2(Ok(buf2));
1331 assert_eq!(err_count.load(Ordering::Relaxed), 1);
1333 assert_eq!(ok_count.load(Ordering::Relaxed), 0);
1334 }
1335
1336 #[test]
1337 fn test_child_block_service() {
1338 let mut device_data = vec![0u8; 16384];
1339 device_data[4096..8192].fill(0x55);
1340 let parent_service = Arc::new(FakeBlockService::new(device_data));
1341
1342 let mappings = Extents::try_new([Extent::new(0..4096, Some(4096))], 0).unwrap();
1343 let file = Arc::new(File::new(mappings, 4096, Transform::None));
1344
1345 let child_service = ChildBlockService::new(parent_service.clone(), file.clone());
1346 assert_eq!(child_service.file().extents().iter_extents(0).count(), 1);
1347
1348 let dest = child_service.allocate_buffer(4096);
1349 let (tx, rx) = std::sync::mpsc::channel();
1350 child_service
1351 .read_blocks(
1352 0,
1353 dest,
1354 Box::new(move |res| {
1355 tx.send(res).unwrap();
1356 }),
1357 )
1358 .unwrap();
1359
1360 let buffer = rx.recv().unwrap().unwrap();
1361 assert_eq!(buffer.len(), 4096);
1362 assert!(buffer.as_ptr_slice().iter_as::<u8>().all(|b| b == 0x55));
1363 }
1364
1365 #[test]
1366 fn test_read_buffer_from_extents_non_aligned_start() {
1367 let base_device_offset = 17408u64; let mut device_data = vec![0u8; 32768];
1369 device_data[base_device_offset as usize..(base_device_offset as usize + 4096)].fill(0x77);
1370 let service = Arc::new(FakeBlockService::new(device_data));
1371
1372 let mappings =
1373 Extents::try_new([Extent::new(0..4096, Some(base_device_offset))], base_device_offset)
1374 .unwrap();
1375
1376 let dest = service.allocate_buffer(4096);
1377 let (tx, rx) = std::sync::mpsc::channel();
1378 read_buffer_from_extents(&mappings, 0, dest, service.as_ref(), move |res| {
1379 tx.send(res).unwrap();
1380 })
1381 .unwrap();
1382
1383 let buffer = rx.recv().unwrap().unwrap();
1384 assert_eq!(buffer.len(), 4096);
1385 assert!(buffer.as_ptr_slice().iter_as::<u8>().all(|b| b == 0x77));
1386 }
1387}