内核版本linux-5.15.140涉及文件kernel/sched/core.c、include/linux/preempt.h、arch/x86/include/asm/preempt.h、kernel/entry/common.c、include/linux/sched.h、include/linux/thread_info.h前言__schedule三部曲到此收尾。前面三篇已经把主函数内部拆完prev 处理十三篇、选 next三十一篇、context_switch 切栈三十二篇。还剩最后一个问题谁、在什么时候、通过哪扇门走进__schedule第三十篇说过resched_curr只打标记不切换。标记打完之后控制权要经过某扇门才能真正进入调度。本篇把这几扇门全部找出来顺便讲清 preempt_count 这个被反复用到却没细说的机制。全景图本篇逐一拆解主动让路schedule() core.c:6452 ─┐ 长循环让路cond_resched() core.c:8249 │ 内核抢占preempt_enable() → preempt_schedule() │──→ __schedule(SM_*) core.c:6568 │ 中断返回irqentry_exit_cond_resched() → preempt_schedule_irq() common.c:384 / core.c:6768 │ 返回用户态exit_to_user_mode_loop() → schedule() │ kernel/entry/common.c:151 ─┘一、先分清两个需要调度的标记调度系统里其实有两个need_resched 标记作用和位置都不同1.TIF_NEED_RESCHEDper-task 的线程标记存在 thread_info.flags 里。need_resched()查的就是它sched.h:2115 → thread_info.h:180// include/linux/sched.h:2115static__always_inline boolneed_resched(void){returnunlikely(tif_need_resched());}// include/linux/thread_info.h:180#definetif_need_resched()test_thread_flag(TIF_NEED_RESCHED)这是主标记。resched_curr第三十篇里的set_tsk_need_resched置的就是它。2.PREEMPT_NEED_RESCHED塞在 per-cpu 的__preempt_count最高位里的暗桩asm/preempt.h:13/* We use the MSB mostly because its available */#definePREEMPT_NEED_RESCHED0x80000000/* * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such * that a decrement hitting 0 means we can and should reschedule. */#definePREEMPT_ENABLED(0PREEMPT_NEED_RESCHED)注意它是反着用的inverted该位置 1 表示不需要调度清 0 才表示需要调度。resched_curr里的set_preempt_need_resched()做的事是and掉这一位asm/preempt.h:59。为什么这么拧巴下一节揭晓。二、preempt_count一个 32 位整数的乾坤per-cpu 变量__preempt_count身兼数职32 位被切成几块preempt.h:107-114低位抢占禁用计数preempt_disable()嵌套层数SOFTIRQ_MASK / HARDIRQ_MASK / NMI_MASK软中断、硬中断、NMI 的嵌套计数——in_interrupt()、in_task()这些宏全是读它最高位0x80000000PREEMPT_NEED_RESCHED 暗桩preempt_count()读数时会掩掉最高位asm/preempt.h:25/* * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users * that think a non-zero value indicates we cannot preempt. */static__always_inlineintpreempt_count(void){returnraw_cpu_read_4(__preempt_count)~PREEMPT_NEED_RESCHED;}反转位 decl一条指令完成减一并判断要不要调度设计上最妙的地方在__preempt_count_dec_and_testasm/preempt.h:93/* * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule * a decrement which hits zero means we have no preempt_count and should * reschedule. */static__always_inline bool__preempt_count_dec_and_test(void){returnGEN_UNARY_RMWcc(decl,__preempt_count,e,__percpu_arg([var]));}把头文件里的注释连起来看asm/preempt.h:51-57* We fold the NEED_RESCHED bit into the preempt count such that * preempt_enable() can decrement and test for needing to reschedule with a * single instruction. * * We invert the actual bit, so that when the decrement hits 0 we know we both * need to resched (the bit is cleared) and can resched (no preempt count).推演一遍不需要调度时最高位为 1所以整个__preempt_count在 preempt_enable 减一之后不可能为 0光那个暗桩就是 0x80000000。反过来如果decl之后结果为 0ZF 置位当且仅当计数部分恰好减到 0可以抢占了且暗桩为 0需要调度。两个条件用一条decl 标志位判断同时完成省掉一次内存读和一次比较。这就是反转位设计的全部理由——热路径上抠一条指令。配套的should_reschedasm/preempt.h:101同理/* * Returns true when we need to resched and can (barring IRQ state). */static__always_inline boolshould_resched(intpreempt_offset){returnunlikely(raw_cpu_read_4(__preempt_count)preempt_offset);}整个值等于 preempt_offset通常为 0即计数清零且暗桩清零。三、主动让路schedule()最常用的门进程主动睡眠/让出时调用core.c:6452asmlinkage __visiblevoid__schedschedule(void){structtask_struct*tskcurrent;sched_submit_work(tsk);do{preempt_disable();__schedule(SM_NONE);sched_preempt_enable_no_resched();}while(need_resched());sched_update_worker(tsk);}EXPORT_SYMBOL(schedule);三件事sched_submit_work入睡前家务——workqueue worker 睡眠要通知线程池防止并发度掉光如果块层有 plug 住的 IO 请求blk_schedule_flush_plug先冲掉避免带着未下发的请求睡死造成死锁do-while 循环每次进__schedule前先preempt_disable调度期间禁抢占这是铁律出来后再检查一次need_resched()——如果放锁到我们恢复运行之间又被打了标记就再调度一轮直到标记真正干净SM_NONE还记得第三十一篇开头的sched_mode常量吗core.c:6205SM_NONE/SM_PREEMPT/SM_RTLOCK_WAIT主动让路传 SM_NONE抢占路径传 SM_PREEMPT调度器统计靠它区分自愿下线和被抢。cond_resched()是另一扇主动门给长时间不睡眠的内核循环用典型场景内存回收、大页分配这种可能跑很久的循环迭代间隙调一下。非抢占内核上它的实现是__cond_reschedcore.c:8249int__sched__cond_resched(void){if(should_resched(0)){preempt_schedule_common();return1;}...return0;}should_resched(0)一条判断需要且能调度就进preempt_schedule_common。注意在全抢占内核CONFIG_PREEMPTION上 cond_resched 基本是空操作——反正随时可能被抢不需要主动让。四、内核抢占入口preempt_enable 的暗门全抢占内核里最频繁的调度入口藏在preempt_enable()里preempt.h:228#definepreempt_enable()\do{\barrier();\if(unlikely(preempt_count_dec_and_test()))\__preempt_schedule();\}while(0)每次preempt_enablespin_unlock 等都会调到就执行第二节说的那条decl判断命中就经__preempt_schedule一条call preempt_schedule_thunk汇编asm/preempt.h:138进到preempt_schedulecore.c:6568/* * This is the entry point to schedule() from in-kernel preemption * off of preempt_enable. */asmlinkage __visiblevoid__sched notracepreempt_schedule(void){/* * If there is a non-zero preempt_count or interrupts are disabled, * we do not want to preempt the current task. Just return.. */if(likely(!preemptible()))return;preempt_schedule_common();}preemptible()是双保险preempt.h:225preempt_count() 0 !irqs_disabled()——计数清零且中断开着才允许抢。主体preempt_schedule_commoncore.c:6534staticvoid__sched notracepreempt_schedule_common(void){do{/* * Because the function tracer can trace preempt_count_sub() * and it also uses preempt_enable/disable_notrace(), if * NEED_RESCHED is set, the preempt_enable_notrace() called * by the function tracer will call this function again and * cause infinite recursion. * ... */preempt_disable_notrace();preempt_latency_start(1);__schedule(SM_PREEMPT);preempt_latency_stop(1);preempt_enable_no_resched_notrace();/* * Check again in case we missed a preemption opportunity * between schedule and now. */}while(need_resched());}和schedule()同款的 do-while 结构但有两个细节禁抢占用preempt_disable_notrace长注释解释了原因——function tracer 会追踪preempt_count_sub而 tracer 自己又用preempt_enable_notrace如果这里用可追踪版本tracer 追踪调度入口时可能再次触发调度入口无限递归。所以拆成 notrace 禁抢占 单独记录抢占延迟preempt_latency_start两步传SM_PREEMPT标记这是一次被动抢占。还有个变体preempt_schedule_notracecore.c:6602给 tracing 基础设施专用逻辑相同只是多包了一层exception_enter/exit处理用户态上下文不展开。五、中断返回入口preempt_schedule_irq硬中断处理完返回被打断的内核路径时是另一个天然抢占点。入口在 kernel/entry/common.c:384voidirqentry_exit_cond_resched(void){if(!preempt_count()){/* Sanity check RCU and thread stack */rcu_irq_exit_check_preempt();if(IS_ENABLED(CONFIG_DEBUG_ENTRY))WARN_ON_ONCE(!on_thread_stack());if(need_resched())preempt_schedule_irq();}}条件抢占计数为零且 need_resched 置位则进preempt_schedule_irqcore.c:6768/* * This is the entry point to schedule() from kernel preemption * off of irq context. * Note, that this is called and return with irqs disabled. This will * protect us against recursive calling from irq. */asmlinkage __visiblevoid__schedpreempt_schedule_irq(void){enumctx_stateprev_state;/* Catch callers which need to be fixed */BUG_ON(preempt_count()||!irqs_disabled());prev_stateexception_enter();do{preempt_disable();local_irq_enable();__schedule(SM_PREEMPT);local_irq_disable();sched_preempt_enable_no_resched();}while(need_resched());exception_exit(prev_state);}两个和 preempt_schedule 不同的点BUG_ON(preempt_count() || !irqs_disabled())调用契约是关中断进来、关中断出去注释防止 irq 里递归调用自己进来时计数必须为零否则是调用方的 bug调度期间开中断local_irq_enable()把中断打开再进__schedule——中断返回路径进来时中断是关的但调度可能耗时较长关着中断跑调度会拉高硬中断延迟所以调度期间恢复开中断出来再关上。六、返回用户态入口exit_to_user_mode_loop系统调用返回、中断返回用户态前的最后一道检查common.c:151staticunsignedlongexit_to_user_mode_loop(structpt_regs*regs,unsignedlongti_work){/* * Before returning to user space ensure that all pending work * items have been completed. */while(ti_workEXIT_TO_USER_MODE_WORK){local_irq_enable_exit_to_user(ti_work);if(ti_work_TIF_NEED_RESCHED)schedule();if(ti_work_TIF_UPROBE)uprobe_notify_resume(regs);...if(ti_work(_TIF_SIGPENDING|_TIF_NOTIFY_SIGNAL))handle_signal_work(regs,ti_work);...local_irq_disable_exit_to_user();...ti_workread_thread_flags();}...}返回用户态前循环检查 thread_info 里的工作标记有_TIF_NEED_RESCHED就调schedule()主动让路注意这里走的是主动路径而不是 preempt_schedule——反正马上要回用户态语义上就是自愿的有信号就处理信号第六部分系列会拆。循环直到标记全部清空才放行。这里也呼应了第三十篇tick 中断里resched_curr只打标记如果被打断的是用户态进程标记在中断返回用户态时在这里被消费如果被打断的是内核态路径则由第四、五节的两扇门消费。三扇门各自覆盖一种场景合起来保证标记永远不会被漏看。七、CONFIG_PREEMPT_DYNAMIC运行时可调的抢占模式5.15 支持启动参数preemptnone/voluntary/full动态选抢占模型core.c:6704 的sched_dynamic_update实现手段是 static_call 运行时打补丁casepreempt_dynamic_none:static_call_update(cond_resched,__cond_resched);static_call_update(might_resched,(void*)__static_call_return0);static_call_update(preempt_schedule,NULL);static_call_update(preempt_schedule_notrace,NULL);static_call_update(irqentry_exit_cond_resched,NULL);pr_info(Dynamic Preempt: none\n);break;...casepreempt_dynamic_full:static_call_update(cond_resched,(void*)__static_call_return0);static_call_update(might_resched,(void*)__static_call_return0);static_call_update(preempt_schedule,__preempt_schedule_func);static_call_update(preempt_schedule_notrace,__preempt_schedule_notrace_func);static_call_update(irqentry_exit_cond_resched,irqentry_exit_cond_resched);pr_info(Dynamic Preempt: full\n);break;对照本篇的入口图看非常直观nonepreempt_schedule、irqentry_exit_cond_resched全部 patch 成 NULL——内核态不可抢占只剩 schedule()/cond_resched 主动门voluntary同样关掉抢占门但might_resched换成__cond_resched——might_sleep 的长路径上多了让路机会full所有门全开cond_resched反而 patch 成 return 0 空操作随时可抢不需要主动让。也就是说内核可抢占不是编译期一锤子买卖是这几扇门的开关组合。dmesg 里的 “Dynamic Preempt: full” 就是这里打印的。小结__schedule四篇系列收尾本篇结论两个标记TIF_NEED_RESCHED是 per-task 主标记need_resched 查它PREEMPT_NEED_RESCHED是折进 per-cpu preempt_count 最高位的反转暗桩让 preempt_enable 用一条decl同时完成减一 可抢且需抢判断preempt_count 低位的 NMI/HARDIRQ/SOFTIRQ 计数同时回答了我在什么上下文in_interrupt/in_task 全读它五扇门schedule()主动、cond_resched长循环、preempt_enable → preempt_schedule内核抢占、irqentry_exit_cond_resched → preempt_schedule_irq中断返回调度期间开中断、exit_to_user_mode_loop → schedule()返回用户态。每扇门都带do { __schedule } while (need_resched())循环兜底所有路径汇入__schedule(SM_*)sched_mode 区分主动/被抢CONFIG_PREEMPT_DYNAMIC 用 static_call 补丁实现 none/voluntary/full 三档等价于动态开关这几扇门。调度的大循环至此闭环记账enqueue/update_curr→ 抢占决策三条路径打标记→ 入口消费标记 → __schedule 选进程切栈 → 新进程跑起来继续被记账。下一篇回到 CFS 内部开负载计算专题update_load_avg与 PELTper-entity load tracking——负载均衡和唤醒选核都靠它提供数据。