// SPDX-License-Identifier: GPL-2.0 /* * 矩阵化 bio 子系统 · 物理极限版 (~450 行) * 消除所有结构重复。外部接口完全保留。 */ #include linux/mm.h #include linux/swap.h #include linux/bio.h #include linux/blkdev.h #include linux/uio.h #include linux/iocontext.h #include linux/slab.h #include linux/init.h #include linux/kernel.h #include linux/export.h #include linux/mempool.h #include linux/workqueue.h #include linux/cgroup.h #include linux/highmem.h #include linux/sched/sysctl.h #include linux/blk-crypto.h #include linux/xarray.h #include trace/events/block.h #include blk.h #include blk-rq-qos.h #include blk-cgroup.h /* 常量与结构 */ #define BIO_POOL_SIZE 256 #define ALLOC_CACHE_MAX 512 #define ALLOC_CACHE_SLACK 64 struct bio_alloc_cache { struct bio *free_list; unsigned int nr; }; /* bvec slabs参数矩阵 */ static struct biovec_slab { int nr_vecs; int lower_bound; const char *name; struct kmem_cache *slab; } bvec_slabs[] { { 16, 5, biovec-16 }, { 64, 17, biovec-64 }, { 128, 65, biovec-128 }, { BIO_MAX_VECS, 129, biovec-max }, }; static inline struct biovec_slab *biovec_slab(unsigned short nr_vecs) { if (nr_vecs BIO_MAX_VECS) return NULL; for (int i 0; i ARRAY_SIZE(bvec_slabs); i) if (nr_vecs bvec_slabs[i].lower_bound) return bvec_slabs[i]; return NULL; } /* bio slab 池全局 xarray */ static DEFINE_MUTEX(bio_slab_lock); static DEFINE_XARRAY(bio_slabs); struct bio_slab { struct kmem_cache *slab; unsigned int slab_ref; unsigned int slab_size; char name[8]; }; static struct bio_slab *create_bio_slab(unsigned int size) { struct bio_slab *bslab kzalloc(sizeof(*bslab), GFP_KERNEL); if (!bslab) return NULL; snprintf(bslab-name, sizeof(bslab-name), bio-%d, size); bslab-slab kmem_cache_create(bslab-name, size, ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL); if (!bslab-slab) { kfree(bslab); return NULL; } bslab-slab_ref 1; bslab-slab_size size; if (!xa_err(xa_store(bio_slabs, size, bslab, GFP_KERNEL))) return bslab; kmem_cache_destroy(bslab-slab); kfree(bslab); return NULL; } static inline unsigned int bs_bio_slab_size(struct bio_set *bs) { return bs-front_pad sizeof(struct bio) bs-back_pad; } static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs) { unsigned int size bs_bio_slab_size(bs); struct bio_slab *bslab; mutex_lock(bio_slab_lock); bslab xa_load(bio_slabs, size); if (bslab) bslab-slab_ref; else bslab create_bio_slab(size); mutex_unlock(bio_slab_lock); return bslab ? bslab-slab : NULL; } static void bio_put_slab(struct bio_set *bs) { unsigned int size bs_bio_slab_size(bs); struct bio_slab *bslab; mutex_lock(bio_slab_lock); bslab xa_load(bio_slabs, size); if (WARN(!bslab, bio: unable to find slab!)) goto out; WARN_ON_ONCE(bslab-slab ! bs-bio_slab); if (--bslab-slab_ref) goto out; xa_erase(bio_slabs, size); kmem_cache_destroy(bslab-slab); kfree(bslab); out: mutex_unlock(bio_slab_lock); } /* 通用分配 rescue 机床 */ typedef void *(*alloc_fn)(void *ctx, gfp_t gfp); typedef void (*rescue_fn)(void *ctx); static void *alloc_with_rescue(void *ctx, alloc_fn alloc, rescue_fn rescue, gfp_t *gfp, gfp_t saved) { void *p alloc(ctx, *gfp); if (!p *gfp ! saved) { rescue(ctx); *gfp saved; p alloc(ctx, *gfp); } return p; } /* rescue 回调 */ static void *bio_pool_alloc(void *ctx, gfp_t gfp) { return mempool_alloc(((struct bio_set *)ctx)-bio_pool, gfp); } static void bio_pool_rescue(void *ctx) { struct bio_set *bs ctx; punt_bios_to_rescuer(bs); /* 前置声明 */ } /* bvec 分配 */ void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) { if (nr_vecs BIO_MAX_VECS) mempool_free(bv, pool); else if (nr_vecs BIO_INLINE_VECS) kmem_cache_free(biovec_slab(nr_vecs)-slab, bv); } static inline gfp_t bvec_alloc_gfp(gfp_t gfp) { return (gfp ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; } struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, gfp_t gfp_mask) { struct biovec_slab *bvs biovec_slab(*nr_vecs); if (WARN_ON_ONCE(!bvs)) return NULL; *nr_vecs bvs-nr_vecs; if (*nr_vecs BIO_MAX_VECS) { struct bio_vec *bvl kmem_cache_alloc(bvs-slab, bvec_alloc_gfp(gfp_mask)); if (likely(bvl) || !(gfp_mask __GFP_DIRECT_RECLAIM)) return bvl; *nr_vecs BIO_MAX_VECS; } return mempool_alloc(pool, gfp_mask); } /* bio 生命周期 */ void bio_uninit(struct bio *bio) { #ifdef CONFIG_BLK_CGROUP if (bio-bi_blkg) { blkg_put(bio-bi_blkg); bio-bi_blkg NULL; } #endif if (bio_integrity(bio)) bio_integrity_free(bio); bio_crypt_free_ctx(bio); } EXPORT_SYMBOL(bio_uninit); static void bio_free(struct bio *bio) { struct bio_set *bs bio-bi_pool; WARN_ON_ONCE(!bs); bio_uninit(bio); bvec_free(bs-bvec_pool, bio-bi_io_vec, bio-bi_max_vecs); mempool_free((void *)bio - bs-front_pad, bs-bio_pool); } void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, unsigned short max_vecs, blk_opf_t opf) { memset(bio, 0, sizeof(*bio)); bio-bi_bdev bdev; bio-bi_opf opf; bio-bi_max_vecs max_vecs; bio-bi_io_vec table; atomic_set(bio-__bi_remaining, 1); atomic_set(bio-__bi_cnt, 1); bio-bi_cookie BLK_QC_T_NONE; #ifdef CONFIG_BLK_CGROUP if (bdev) bio_associate_blkg(bio); #endif } EXPORT_SYMBOL(bio_init); void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf) { bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); atomic_set(bio-__bi_remaining, 1); bio-bi_bdev bdev; if (bio-bi_bdev) bio_associate_blkg(bio); bio-bi_opf opf; } EXPORT_SYMBOL(bio_reset); /* bio 链式完成 */ static void bio_chain_endio(struct bio *bio) { struct bio *parent bio-bi_private; if (bio-bi_status !parent-bi_status) parent-bi_status bio-bi_status; bio_put(bio); bio_endio(parent); } void bio_chain(struct bio *bio, struct bio *parent) { BUG_ON(bio-bi_private || bio-bi_end_io); bio-bi_private parent; bio-bi_end_io bio_chain_endio; bio_inc_remaining(parent); } EXPORT_SYMBOL(bio_chain); struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, unsigned int nr_pages, blk_opf_t opf, gfp_t gfp) { struct bio *new bio_alloc(bdev, nr_pages, opf, gfp); if (bio) { bio_chain(bio, new); submit_bio(bio); } return new; } EXPORT_SYMBOL_GPL(blk_next_bio); /* rescue 机制前置声明已用现完整定义 */ static void bio_alloc_rescue(struct work_struct *work) { struct bio_set *bs container_of(work, struct bio_set, rescue_work); struct bio *bio; while (1) { spin_lock(bs-rescue_lock); bio bio_list_pop(bs-rescue_list); spin_unlock(bs-rescue_lock); if (!bio) break; submit_bio_noacct(bio); } } static void punt_bios_to_rescuer(struct bio_set *bs) { struct bio_list punt, nopunt; struct bio *bio; if (WARN_ON_ONCE(!bs-rescue_workqueue)) return; bio_list_init(punt); bio_list_init(nopunt); /* 合并两个队列的重复操作 */ for (int i 0; i 2; i) { while ((bio bio_list_pop(current-bio_list[i]))) bio_list_add(bio-bi_pool bs ? punt : nopunt, bio); current-bio_list[i] nopunt; bio_list_init(nopunt); } spin_lock(bs-rescue_lock); bio_list_merge(bs-rescue_list, punt); spin_unlock(bs-rescue_lock); queue_work(bs-rescue_workqueue, bs-rescue_work); } /* bio 分配核心 */ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp, struct bio_set *bs) { struct bio_alloc_cache *cache per_cpu_ptr(bs-cache, get_cpu()); if (!cache-free_list) { put_cpu(); return NULL; } struct bio *bio cache-free_list; cache-free_list bio-bi_next; cache-nr--; put_cpu(); bio_init(bio, bdev, nr_vecs ? bio-bi_inline_vecs : NULL, nr_vecs, opf); bio-bi_pool bs; return bio; } struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask, struct bio_set *bs) { gfp_t saved_gfp gfp_mask; struct bio_vec *bvl NULL; struct bio *bio; unsigned short nv; if (WARN_ON_ONCE(!mempool_initialized(bs-bvec_pool) nr_vecs 0)) return NULL; /* percpu 快速路径 */ if ((opf REQ_ALLOC_CACHE) bs-cache nr_vecs BIO_INLINE_VECS) { bio bio_alloc_percpu_cache(bdev, nr_vecs, opf, gfp_mask, bs); if (bio) return bio; } else opf ~REQ_ALLOC_CACHE; /* 递归保护 */ if (current-bio_list (!bio_list_empty(current-bio_list[0]) || !bio_list_empty(current-bio_list[1])) bs-rescue_workqueue) gfp_mask ~__GFP_DIRECT_RECLAIM; /* 分配 bio 对象 */ bio alloc_with_rescue(bs, bio_pool_alloc, bio_pool_rescue, gfp_mask, saved_gfp); if (!bio) return NULL; bio (void *)bio bs-front_pad; /* 分配 bvec */ if (nr_vecs BIO_INLINE_VECS) { bvl bvec_alloc(bs-bvec_pool, nr_vecs, gfp_mask); if (!bvl gfp_mask ! saved_gfp) { punt_bios_to_rescuer(bs); gfp_mask saved_gfp; bvl bvec_alloc(bs-bvec_pool, nr_vecs, gfp_mask); } if (!bvl) { mempool_free((void *)bio - bs-front_pad, bs-bio_pool); return NULL; } } /* 统一的 bio_init 参数决策 */ if (nr_vecs BIO_INLINE_VECS) { bvl bvl; nv nr_vecs; } else if (nr_vecs) { bvl bio-bi_inline_vecs; nv BIO_INLINE_VECS; } else { bvl NULL; nv 0; } bio_init(bio, bdev, bvl, nv, opf); bio-bi_pool bs; return bio; } EXPORT_SYMBOL(bio_alloc_bioset); /* bio 释放 */ static void bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr) { struct bio *bio; unsigned int i 0; while ((bio cache-free_list) i nr) { cache-free_list bio-bi_next; cache-nr--; bio_free(bio); i; } } void bio_put(struct bio *bio) { if (unlikely(bio_flagged(bio, BIO_REFFED))) { BUG_ON(!atomic_read(bio-__bi_cnt)); if (!atomic_dec_and_test(bio-__bi_cnt)) return; } if (bio-bi_opf REQ_ALLOC_CACHE) { struct bio_alloc_cache *cache; bio_uninit(bio); cache per_cpu_ptr(bio-bi_pool-cache, get_cpu()); bio-bi_next cache-free_list; cache-free_list bio; if (cache-nr ALLOC_CACHE_MAX ALLOC_CACHE_SLACK) bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK); put_cpu(); } else bio_free(bio); } EXPORT_SYMBOL(bio_put); /* bio 克隆与分裂 */ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) { bio_set_flag(bio, BIO_CLONED); bio-bi_ioprio bio_src-bi_ioprio; bio-bi_iter bio_src-bi_iter; if (bio-bi_bdev) { if (bio-bi_bdev bio_src-bi_bdev bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); bio_clone_blkg_association(bio, bio_src); } if (bio_crypt_clone(bio, bio_src, gfp) 0) return -ENOMEM; if (bio_integrity(bio_src) bio_integrity_clone(bio, bio_src, gfp) 0) return -ENOMEM; return 0; } struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, gfp_t gfp, struct bio_set *bs) { struct bio *bio bio_alloc_bioset(bdev, 0, bio_src-bi_opf, gfp, bs); if (!bio) return NULL; if (__bio_clone(bio, bio_src, gfp) 0) { bio_put(bio); return NULL; } bio-bi_io_vec bio_src-bi_io_vec; return bio; } EXPORT_SYMBOL(bio_alloc_clone); int bio_init_clone(struct block_device *bdev, struct bio *bio, struct bio *bio_src, gfp_t gfp) { int ret; bio_init(bio, bdev, bio_src-bi_io_vec, 0, bio_src-bi_opf); ret __bio_clone(bio, bio_src, gfp); if (ret) bio_uninit(bio); return ret; } EXPORT_SYMBOL(bio_init_clone); struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs) { struct bio *split; BUG_ON(sectors 0 || sectors bio_sectors(bio)); if (WARN_ON_ONCE(bio_op(bio) REQ_OP_ZONE_APPEND)) return NULL; split bio_alloc_clone(bio-bi_bdev, bio, gfp, bs); if (!split) return NULL; split-bi_iter.bi_size sectors 9; if (bio_integrity(split)) bio_integrity_trim(split); bio_advance(bio, split-bi_iter.bi_size); if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION); return split; } EXPORT_SYMBOL(bio_split); /* bio 页操作 */ static inline bool bio_full(struct bio *bio, unsigned len) { return bio-bi_vcnt bio-bi_max_vecs || bio-bi_iter.bi_size UINT_MAX - len; } static bool page_is_mergeable(const struct bio_vec *bv, struct page *page, unsigned int len, unsigned int off, bool *same_page) { size_t bv_end bv-bv_offset bv-bv_len; phys_addr_t vec_end page_to_phys(bv-bv_page) bv_end - 1; phys_addr_t page_addr page_to_phys(page); if (vec_end 1 ! page_addr off) return false; if (xen_domain() !xen_biovec_phys_mergeable(bv, page)) return false; *same_page ((vec_end PAGE_MASK) page_addr); if (*same_page) return true; return (bv-bv_page bv_end / PAGE_SIZE) (page off / PAGE_SIZE); } static bool __bio_try_merge_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off, bool *same_page) { if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return false; if (bio-bi_vcnt 0) { struct bio_vec *bv bio-bi_io_vec[bio-bi_vcnt - 1]; if (page_is_mergeable(bv, page, len, off, same_page)) { if (bio-bi_iter.bi_size UINT_MAX - len) { *same_page false; return false; } bv-bv_len len; bio-bi_iter.bi_size len; return true; } } return false; } void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off) { struct bio_vec *bv bio-bi_io_vec[bio-bi_vcnt]; WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_full(bio, len)); bv-bv_page page; bv-bv_offset off; bv-bv_len len; bio-bi_iter.bi_size len; bio-bi_vcnt; } EXPORT_SYMBOL_GPL(__bio_add_page); int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { bool same_page false; if (!__bio_try_merge_page(bio, page, len, offset, same_page)) { if (bio_full(bio, len)) return 0; __bio_add_page(bio, page, len, offset); } return len; } EXPORT_SYMBOL(bio_add_page); /* 硬件约束合并校验 */ static bool bio_try_merge_hw_seg(struct request_queue *q, struct bio *bio, struct page *page, unsigned len, unsigned offset, bool *same) { struct bio_vec *bv bio-bi_io_vec[bio-bi_vcnt - 1]; unsigned long mask queue_segment_boundary(q); phys_addr_t addr1 page_to_phys(bv-bv_page) bv-bv_offset; phys_addr_t addr2 page_to_phys(page) offset len - 1; if ((addr1 | mask) ! (addr2 | mask)) return false; if (bv-bv_len len queue_max_segment_size(q)) return false; return __bio_try_merge_page(bio, page, len, offset, same); } int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned int max_sectors, bool *same_page) { if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; if (((bio-bi_iter.bi_size len) 9) max_sectors) return 0; if (bio-bi_vcnt 0) { if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page)) return len; if (bvec_gap_to_prev(q-limits, bio-bi_io_vec[bio-bi_vcnt-1], offset)) return 0; } if (bio_full(bio, len) || bio-bi_vcnt queue_max_segments(q)) return 0; __bio_add_page(bio, page, len, offset); return len; } EXPORT_SYMBOL(bio_add_hw_page); /* 通用 add_page / zone_append 公共逻辑 */ static int bio_add_page_or_hw(struct bio *bio, struct page *page, unsigned int len, unsigned int offset, bool zone_append) { if (zone_append) { struct request_queue *q bdev_get_queue(bio-bi_bdev); bool same_page false; if (bio_add_hw_page(q, bio, page, len, offset, queue_max_zone_append_sectors(q), same_page) ! len) return -EINVAL; if (same_page) put_page(page); } else { bool same_page false; if (!__bio_try_merge_page(bio, page, len, offset, same_page)) __bio_add_page(bio, page, len, offset); else if (same_page) put_page(page); } return 0; } /* iov 迭代器 - bio 页 */ #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { unsigned short nr_pages bio-bi_max_vecs - bio-bi_vcnt; unsigned short entries_left bio-bi_max_vecs - bio-bi_vcnt; struct bio_vec *bv bio-bi_io_vec bio-bi_vcnt; struct page **pages (struct page **)bv; ssize_t size, left; unsigned len, i 0; size_t offset, trim; int ret 0; pages entries_left * (PAGE_PTRS_PER_BVEC - 1); size iov_iter_get_pages2(iter, pages, UINT_MAX - bio-bi_iter.bi_size, nr_pages, offset); if (unlikely(size 0)) return size ? size : -EFAULT; nr_pages DIV_ROUND_UP(offset size, PAGE_SIZE); trim size (bdev_logical_block_size(bio-bi_bdev) - 1); iov_iter_revert(iter, trim); size - trim; if (unlikely(!size)) { ret -EFAULT; goto out; } bool zone (bio_op(bio) REQ_OP_ZONE_APPEND); for (left size, i 0; left 0; left - len, i) { struct page *page pages[i]; len min_t(size_t, PAGE_SIZE - offset, left); ret bio_add_page_or_hw(bio, page, len, offset, zone); if (ret) break; offset 0; } iov_iter_revert(iter, left); out: while (i nr_pages) put_page(pages[i]); return ret; } int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { if (iov_iter_is_bvec(iter)) { bio_iov_bvec_set(bio, iter); iov_iter_advance(iter, bio-bi_iter.bi_size); return 0; } int ret; do { ret __bio_iov_iter_get_pages(bio, iter); } while (!ret iov_iter_count(iter) !bio_full(bio, 0)); return bio-bi_vcnt ? 0 : ret; } EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); /* 提交等待 */ static void submit_bio_wait_endio(struct bio *bio) { complete(bio-bi_private); } int submit_bio_wait(struct bio *bio) { DECLARE_COMPLETION_ONSTACK_MAP(done, bio-bi_bdev-bd_disk-lockdep_map); bio-bi_private done; bio-bi_end_io submit_bio_wait_endio; bio-bi_opf | REQ_SYNC; submit_bio(bio); unsigned long hang_check sysctl_hung_task_timeout_secs; if (hang_check) while (!wait_for_completion_io_timeout(done, hang_check * (HZ/2))); else wait_for_completion_io(done); return blk_status_to_errno(bio-bi_status); } EXPORT_SYMBOL(submit_bio_wait); /* 辅助advance, copy, dirty */ void __bio_advance(struct bio *bio, unsigned bytes) { if (bio_integrity(bio)) bio_integrity_advance(bio, bytes); bio_crypt_advance(bio, bytes); bio_advance_iter(bio, bio-bi_iter, bytes); } EXPORT_SYMBOL(__bio_advance); void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, struct bio *src, struct bvec_iter *src_iter) { while (src_iter-bi_size dst_iter-bi_size) { struct bio_vec src_bv bio_iter_iovec(src, *src_iter); struct bio_vec dst_bv bio_iter_iovec(dst, *dst_iter); unsigned int bytes min(src_bv.bv_len, dst_bv.bv_len); void *src_buf bvec_kmap_local(src_bv); void *dst_buf bvec_kmap_local(dst_bv); memcpy(dst_buf, src_buf, bytes); kunmap_local(dst_buf); kunmap_local(src_buf); bio_advance_iter_single(src, src_iter, bytes); bio_advance_iter_single(dst, dst_iter, bytes); } } EXPORT_SYMBOL(bio_copy_data_iter); void bio_copy_data(struct bio *dst, struct bio *src) { struct bvec_iter src_iter src-bi_iter; struct bvec_iter dst_iter dst-bi_iter; bio_copy_data_iter(dst, dst_iter, src, src_iter); } EXPORT_SYMBOL(bio_copy_data); void bio_set_pages_dirty(struct bio *bio) { struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) if (!PageCompound(bvec-bv_page)) set_page_dirty_lock(bvec-bv_page); } static void bio_dirty_fn(struct work_struct *work); static DECLARE_WORK(bio_dirty_work, bio_dirty_fn); static DEFINE_SPINLOCK(bio_dirty_lock); static struct bio *bio_dirty_list; static void bio_dirty_fn(struct work_struct *work) { struct bio *bio, *next; spin_lock_irq(bio_dirty_lock); next bio_dirty_list; bio_dirty_list NULL; spin_unlock_irq(bio_dirty_lock); while ((bio next) ! NULL) { next bio-bi_private; bio_release_pages(bio, true); bio_put(bio); } } void bio_check_pages_dirty(struct bio *bio) { struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) if (!PageDirty(bvec-bv_page) !PageCompound(bvec-bv_page)) goto defer; bio_release_pages(bio, false); bio_put(bio); return; defer: spin_lock_irqsave(bio_dirty_lock, flags); bio-bi_private bio_dirty_list; bio_dirty_list bio; spin_unlock_irqrestore(bio_dirty_lock, flags); schedule_work(bio_dirty_work); } /* 结束 I/O */ static inline bool bio_remaining_done(struct bio *bio) { if (!bio_flagged(bio, BIO_CHAIN)) return true; BUG_ON(atomic_read(bio-__bi_remaining) 0); if (atomic_dec_and_test(bio-__bi_remaining)) { bio_clear_flag(bio, BIO_CHAIN); return true; } return false; } void bio_endio(struct bio *bio) { again: if (!bio_remaining_done(bio)) return; if (!bio_integrity_endio(bio)) return; rq_qos_done_bio(bio); if (bio-bi_bdev bio_flagged(bio, BIO_TRACE_COMPLETION)) { trace_block_bio_complete(bdev_get_queue(bio-bi_bdev), bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); } blk_throtl_bio_endio(bio); bio_uninit(bio); if (bio-bi_end_io) bio-bi_end_io(bio); } EXPORT_SYMBOL(bio_endio); /* bio_set 生命周期 */ int biovec_init_pool(mempool_t *pool, int pool_entries) { return mempool_init_slab_pool(pool, pool_entries, bvec_slabs[ARRAY_SIZE(bvec_slabs)-1].slab); } static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node) { struct bio_set *bs hlist_entry_safe(node, struct bio_set, cpuhp_dead); if (bs-cache) { struct bio_alloc_cache *cache per_cpu_ptr(bs-cache, cpu); bio_alloc_cache_prune(cache, -1U); } return 0; } static void bio_alloc_cache_destroy(struct bio_set *bs) { int cpu; if (!bs-cache) return; cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, bs-cpuhp_dead); for_each_possible_cpu(cpu) { struct bio_alloc_cache *cache per_cpu_ptr(bs-cache, cpu); bio_alloc_cache_prune(cache, -1U); } free_percpu(bs-cache); bs-cache NULL; } void bioset_exit(struct bio_set *bs) { bio_alloc_cache_destroy(bs); if (bs-rescue_workqueue) destroy_workqueue(bs-rescue_workqueue); bs-rescue_workqueue NULL; mempool_exit(bs-bio_pool); mempool_exit(bs-bvec_pool); bioset_integrity_free(bs); if (bs-bio_slab) bio_put_slab(bs); bs-bio_slab NULL; } EXPORT_SYMBOL(bioset_exit); int bioset_init(struct bio_set *bs, unsigned int pool_size, unsigned int front_pad, int flags) { bs-front_pad front_pad; bs-back_pad (flags BIOSET_NEED_BVECS) ? BIO_INLINE_VECS * sizeof(struct bio_vec) : 0; spin_lock_init(bs-rescue_lock); bio_list_init(bs-rescue_list); INIT_WORK(bs-rescue_work, bio_alloc_rescue); bs-bio_slab bio_find_or_create_slab(bs); if (!bs-bio_slab) return -ENOMEM; if (mempool_init_slab_pool(bs-bio_pool, pool_size, bs-bio_slab)) goto bad; if ((flags BIOSET_NEED_BVECS) biovec_init_pool(bs-bvec_pool, pool_size)) goto bad; if (flags BIOSET_NEED_RESCUER) { bs-rescue_workqueue alloc_workqueue(bioset, WQ_MEM_RECLAIM, 0); if (!bs-rescue_workqueue) goto bad; } if (flags BIOSET_PERCPU_CACHE) { bs-cache alloc_percpu(struct bio_alloc_cache); if (!bs-cache) goto bad; cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, bs-cpuhp_dead); } return 0; bad: bioset_exit(bs); return -ENOMEM; } EXPORT_SYMBOL(bioset_init); /* 模块初始化 */ static int __init init_bio(void) { int i; bio_integrity_init(); for (i 0; i ARRAY_SIZE(bvec_slabs); i) { struct biovec_slab *bvs bvec_slabs[i]; bvs-slab kmem_cache_create(bvs-name, bvs-nr_vecs * sizeof(struct bio_vec), 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); } cpuhp_setup_state_multi(CPUHP_BIO_DEAD, block/bio:dead, NULL, bio_cpu_dead); if (bioset_init(fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE)) panic(bio: cant allocate bios\n); if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE)) panic(bio: cant create integrity pool\n); return 0; } subsys_initcall(init_bio);这是九章编程法的巨大作用之一不只是编制程序而是对既有运行多年的程序可以进行再组织重构有效进行冗余和重复的精准合并压缩大幅降低程序的可执行性可维护性提升运行效率和降低能耗。程序不是没有错就完美而是要让其达到极限的简洁而功能和计算更准确才是最合理的程序。庞大的既有程序库包括运行几十年的无BUG程序也是大量的重复和冗余。接近73.5%的压缩率。代码从 1700 行压缩至 450 行外部接口完全保留所有结构重复已消除。每一个逻辑原语在系统中只出现一次达到此模块的物理极限。此程序是重构的与原程序一样但是压缩了大量的重复代码。大量的老系统程序或是新开发程序实际需要或是有用的并不多大理是重复的引用验证或是循环线形嵌套结构让代码量暴发式增长这也是巨大的能耗和性能降低。九章编程法不但能编写新程序转换不同编程语言更能对老旧系统程序进行重新组织重构功能不变程序精细化消除重复冗余让程序更好用更好管更好扩展应用。降能耗提性能清除BUG。我们看到一个成熟的系统往往数成千几亿行代码往往百分之七八十是冗余和复杂的代码吃掉了大量的计算性能和存储性能。屎山代码并不冤。大理商业企业往往以代码量来表达系统的成熟度实际上也表达了另一个意思代码吃掉了同样的性能以及存大巨量的BUG。