fengchi_oneplus_13s.patch

diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst
index 0b650bb550e6..3170747226f6 100644
--- a/Documentation/scheduler/index.rst
+++ b/Documentation/scheduler/index.rst
@@ -19,7 +19,6 @@ Scheduler
     sched-nice-design
     sched-rt-group
     sched-stats
-    sched-ext
     sched-debug
 
     text_files
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
deleted file mode 100644
index 84c30b44f104..000000000000
--- a/Documentation/scheduler/sched-ext.rst
+++ /dev/null
@@ -1,230 +0,0 @@
-==========================
-Extensible Scheduler Class
-==========================
-
-sched_ext is a scheduler class whose behavior can be defined by a set of BPF
-programs - the BPF scheduler.
-
-* sched_ext exports a full scheduling interface so that any scheduling
-  algorithm can be implemented on top.
-
-* The BPF scheduler can group CPUs however it sees fit and schedule them
-  together, as tasks aren't tied to specific CPUs at the time of wakeup.
-
-* The BPF scheduler can be turned on and off dynamically anytime.
-
-* The system integrity is maintained no matter what the BPF scheduler does.
-  The default scheduling behavior is restored anytime an error is detected,
-  a runnable task stalls, or on invoking the SysRq key sequence
-  :kbd:`SysRq-S`.
-
-Switching to and from sched_ext
-===============================
-
-``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and
-``tools/sched_ext`` contains the example schedulers.
-
-sched_ext is used only when the BPF scheduler is loaded and running.
-
-If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be
-treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is
-loaded. On load, such tasks will be switched to and scheduled by sched_ext.
-
-The BPF scheduler can choose to schedule all normal and lower class tasks by
-calling ``scx_bpf_switch_all()`` from its ``init()`` operation. In this
-case, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE`` and
-``SCHED_EXT`` tasks are scheduled by sched_ext. In the example schedulers,
-this mode can be selected with the ``-a`` option.
-
-Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or
-detection of any internal error including stalled runnable tasks aborts the
-BPF scheduler and reverts all tasks back to CFS.
-
-.. code-block:: none
-
-    # make -j16 -C tools/sched_ext
-    # tools/sched_ext/scx_example_simple
-    local=0 global=3
-    local=5 global=24
-    local=9 global=44
-    local=13 global=56
-    local=17 global=72
-    ^CEXIT: BPF scheduler unregistered
-
-If ``CONFIG_SCHED_DEBUG`` is set, the current status of the BPF scheduler
-and whether a given task is on sched_ext can be determined as follows:
-
-.. code-block:: none
-
-    # cat /sys/kernel/debug/sched/ext
-    ops                           : simple
-    enabled                       : 1
-    switching_all                 : 1
-    switched_all                  : 1
-    enable_state                  : enabled
-
-    # grep ext /proc/self/sched
-    ext.enabled                                  :                    1
-
-The Basics
-==========
-
-Userspace can implement an arbitrary BPF scheduler by loading a set of BPF
-programs that implement ``struct sched_ext_ops``. The only mandatory field
-is ``ops.name`` which must be a valid BPF object name. All operations are
-optional. The following modified excerpt is from
-``tools/sched/scx_example_simple.bpf.c`` showing a minimal global FIFO
-scheduler.
-
-.. code-block:: c
-
-    s32 BPF_STRUCT_OPS(simple_init)
-    {
-            if (!switch_partial)
-                    scx_bpf_switch_all();
-            return 0;
-    }
-
-    void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
-    {
-            if (enq_flags & SCX_ENQ_LOCAL)
-                    scx_bpf_dispatch(p, SCX_DSQ_LOCAL, enq_flags);
-            else
-                    scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, enq_flags);
-    }
-
-    void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei)
-    {
-            exit_type = ei->type;
-    }
-
-    SEC(".struct_ops")
-    struct sched_ext_ops simple_ops = {
-            .enqueue                = (void *)simple_enqueue,
-            .init                   = (void *)simple_init,
-            .exit                   = (void *)simple_exit,
-            .name                   = "simple",
-    };
-
-Dispatch Queues
----------------
-
-To match the impedance between the scheduler core and the BPF scheduler,
-sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a
-priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``),
-and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage
-an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and
-``scx_bpf_destroy_dsq()``.
-
-A CPU always executes a task from its local DSQ. A task is "dispatched" to a
-DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's
-local DSQ.
-
-When a CPU is looking for the next task to run, if the local DSQ is not
-empty, the first task is picked. Otherwise, the CPU tries to consume the
-global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()``
-is invoked.
-
-Scheduling Cycle
-----------------
-
-The following briefly shows how a waking task is scheduled and executed.
-
-1. When a task is waking up, ``ops.select_cpu()`` is the first operation
-   invoked. This serves two purposes. First, CPU selection optimization
-   hint. Second, waking up the selected CPU if idle.
-
-   The CPU selected by ``ops.select_cpu()`` is an optimization hint and not
-   binding. The actual decision is made at the last step of scheduling.
-   However, there is a small performance gain if the CPU
-   ``ops.select_cpu()`` returns matches the CPU the task eventually runs on.
-
-   A side-effect of selecting a CPU is waking it up from idle. While a BPF
-   scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper,
-   using ``ops.select_cpu()`` judiciously can be simpler and more efficient.
-
-   Note that the scheduler core will ignore an invalid CPU selection, for
-   example, if it's outside the allowed cpumask of the task.
-
-2. Once the target CPU is selected, ``ops.enqueue()`` is invoked. It can
-   make one of the following decisions:
-
-   * Immediately dispatch the task to either the global or local DSQ by
-     calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or
-     ``SCX_DSQ_LOCAL``, respectively.
-
-   * Immediately dispatch the task to a custom DSQ by calling
-     ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63.
-
-   * Queue the task on the BPF side.
-
-3. When a CPU is ready to schedule, it first looks at its local DSQ. If
-   empty, it then looks at the global DSQ. If there still isn't a task to
-   run, ``ops.dispatch()`` is invoked which can use the following two
-   functions to populate the local DSQ.
-
-   * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can
-     be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``,
-     ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()``
-     currently can't be called with BPF locks held, this is being worked on
-     and will be supported. ``scx_bpf_dispatch()`` schedules dispatching
-     rather than performing them immediately. There can be up to
-     ``ops.dispatch_max_batch`` pending tasks.
-
-   * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ
-     to the dispatching DSQ. This function cannot be called with any BPF
-     locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks
-     before trying to consume the specified DSQ.
-
-4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ,
-   the CPU runs the first one. If empty, the following steps are taken:
-
-   * Try to consume the global DSQ. If successful, run the task.
-
-   * If ``ops.dispatch()`` has dispatched any tasks, retry #3.
-
-   * If the previous task is an SCX task and still runnable, keep executing
-     it (see ``SCX_OPS_ENQ_LAST``).
-
-   * Go idle.
-
-Note that the BPF scheduler can always choose to dispatch tasks immediately
-in ``ops.enqueue()`` as illustrated in the above simple example. If only the
-built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as
-a task is never queued on the BPF scheduler and both the local and global
-DSQs are consumed automatically.
-
-``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use
-``scx_bpf_dispatch_vtime()`` for the priority queue. See the function
-documentation and usage in ``tools/sched_ext/scx_example_simple.bpf.c`` for
-more information.
-
-Where to Look
-=============
-
-* ``include/linux/sched/ext.h`` defines the core data structures, ops table
-  and constants.
-
-* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers.
-  The functions prefixed with ``scx_bpf_`` can be called from the BPF
-  scheduler.
-
-* ``tools/sched_ext/`` hosts example BPF scheduler implementations.
-
-  * ``scx_example_simple[.bpf].c``: Minimal global FIFO scheduler example
-    using a custom DSQ.
-
-  * ``scx_example_qmap[.bpf].c``: A multi-level FIFO scheduler supporting
-    five levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``.
-
-ABI Instability
-===============
-
-The APIs provided by sched_ext to BPF schedulers programs have no stability
-guarantees. This includes the ops table callbacks and constants defined in
-``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in
-``kernel/sched/ext.c``.
-
-While we will attempt to provide a relatively stable API surface when
-possible, they are subject to change without warning between kernel
-versions.
diff --git a/MAINTAINERS b/MAINTAINERS
index ff6b9ea9b788..46ca82b0b506 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -19125,8 +19125,6 @@ R:	Ben Segall <bsegall@google.com> (CONFIG_CFS_BANDWIDTH)
 R:	Mel Gorman <mgorman@suse.de> (CONFIG_NUMA_BALANCING)
 R:	Daniel Bristot de Oliveira <bristot@redhat.com> (SCHED_DEADLINE)
 R:	Valentin Schneider <vschneid@redhat.com> (TOPOLOGY)
-R:	Tejun Heo <tj@kernel.org> (SCHED_EXT)
-R:	David Vernet <void@manifault.com> (SCHED_EXT)
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core
@@ -19135,7 +19133,6 @@ F:	include/linux/sched.h
 F:	include/linux/wait.h
 F:	include/uapi/linux/sched.h
 F:	kernel/sched/
-F:	tools/sched_ext/
 
 SCSI LIBSAS SUBSYSTEM
 R:	John Garry <john.g.garry@oracle.com>
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index f93980c09d7f..5ba590235a8c 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -6,8 +6,7 @@ CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BPF_SYSCALL=y
 CONFIG_BPF_JIT=y
 CONFIG_PREEMPT=y
-CONFIG_SLIM_SCHED=y
-CONFIG_SCHED_CLASS_EXT=y
+CONFIG_HMBIRD_SCHED=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
 CONFIG_BSD_PROCESS_ACCT=y
 CONFIG_BSD_PROCESS_ACCT_V3=y
diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig
index 42c199371c79..ef087e70fba4 100644
--- a/arch/arm64/configs/gki_defconfig
+++ b/arch/arm64/configs/gki_defconfig
@@ -10,8 +10,7 @@ CONFIG_BPF_JIT_ALWAYS_ON=y
 # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
 CONFIG_BPF_LSM=y
 CONFIG_PREEMPT=y
-CONFIG_SLIM_SCHED=y
-CONFIG_SCHED_CLASS_EXT=y
+CONFIG_HMBIRD_SCHED=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
 CONFIG_TASKSTATS=y
 CONFIG_TASK_DELAY_ACCT=y
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 4bd028fa7500..5825ce9d6d7b 100755
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -510,6 +510,9 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)
 		}
 
 		sched_set_fifo(ev_thread->worker->task);
+#ifdef CONFIG_HMBIRD_SCHED
+		hmbird_set_sched_prop(ev_thread->worker->task, SCHED_PROP_DEADLINE_LEVEL3);
+#endif
 	}
 
 	ret = drm_vblank_init(ddev, priv->num_crtcs);
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c
index bd001445815e..dcf2e1ebf9c5 100644
--- a/drivers/tty/sysrq.c
+++ b/drivers/tty/sysrq.c
@@ -524,7 +524,6 @@ static const struct sysrq_key_op *sysrq_key_table[62] = {
 	NULL,				/* P */
 	NULL,				/* Q */
 	NULL,				/* R */
-	/* S: May be registered by sched_ext for resetting */
 	NULL,				/* S */
 	NULL,				/* T */
 	NULL,				/* U */
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 3b9dde0c15c0..c0fee44f5417 100755
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -131,7 +131,7 @@
 	*(__dl_sched_class)			\
 	*(__rt_sched_class)			\
 	*(__fair_sched_class)			\
-	*(__ext_sched_class)			\
+	*(__hmbird_sched_class)			\
 	*(__idle_sched_class)			\
 	__sched_class_lowest = .;
 
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 6991b190b8dc..39c797c5cc75 100755
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -73,7 +73,9 @@ struct task_delay_info;
 struct task_group;
 struct user_event_mm;
 
-#include <linux/sched/ext.h>
+#ifdef CONFIG_HMBIRD_SCHED
+#include <linux/sched/hmbird.h>
+#endif
 
 /*
  * Task state bitmask. NOTE! These bits are also
@@ -298,11 +300,6 @@ enum {
 
 extern void scheduler_tick(void);
 
-#ifdef CONFIG_SLIM_SCHED
-extern enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer);
-extern void stop_shadow_tick_timer(void);
-#endif
-
 #define	MAX_SCHEDULE_TIMEOUT		LONG_MAX
 
 extern long schedule_timeout(long timeout);
@@ -1523,13 +1520,8 @@ struct task_struct {
 	 */
 	struct callback_head		l1d_flush_kill;
 #endif
-#ifdef CONFIG_SLIM_SCHED
-	ANDROID_KABI_USE(1, unsigned long sched_prop);
-	ANDROID_KABI_USE(2, struct sched_ext_entity *scx);
-#else
 	ANDROID_KABI_RESERVE(1);
 	ANDROID_KABI_RESERVE(2);
-#endif
 	ANDROID_KABI_RESERVE(3);
 	ANDROID_KABI_RESERVE(4);
 	ANDROID_KABI_RESERVE(5);
@@ -1932,6 +1924,91 @@ static inline int task_nice(const struct task_struct *p)
 	return PRIO_TO_NICE((p)->static_prio);
 }
 
+#ifdef CONFIG_HMBIRD_SCHED
+static inline bool task_is_top_task(struct task_struct *p)
+{
+	return (((struct hmbird_entity *)(p->android_oem_data1[HMBIRD_TS_IDX]))
+		->top_task_prop & TOP_TASK_BITS_MASK);
+}
+
+static inline int get_top_task_prop(struct task_struct *p)
+{
+	return get_hmbird_ts(p)->top_task_prop;
+}
+
+static inline int set_top_task_prop(struct task_struct *p, u64 set, u64 clear)
+{
+	if (set)
+		get_hmbird_ts(p)->top_task_prop |= set;
+	if (clear)
+		get_hmbird_ts(p)->top_task_prop &= ~clear;
+	return 0;
+}
+
+static inline void reset_top_task_prop(struct task_struct *p)
+{
+	get_hmbird_ts(p)->top_task_prop = 0;
+}
+
+static inline int hmbird_set_sched_prop(struct task_struct *p, unsigned long sp)
+{
+	struct hmbird_entity *entity = get_hmbird_ts(p);
+
+	if (entity) {
+		entity->sched_prop = sp;
+	}
+	return 0;
+}
+
+static inline unsigned long hmbird_get_sched_prop(struct task_struct *p)
+{
+	struct hmbird_entity *entity = get_hmbird_ts(p);
+
+	if (entity)
+		return entity->sched_prop;
+	else
+		return 0;
+}
+
+static inline void hmbird_set_dsq_id(struct task_struct *p, unsigned long dsq)
+{
+	unsigned long new_dsq;
+	struct hmbird_entity *entity = get_hmbird_ts(p);
+
+	if (entity) {
+		new_dsq = (entity->sched_prop & ~SCHED_PROP_DEADLINE_MASK) | dsq;
+		entity->sched_prop = new_dsq;
+	}
+}
+
+static inline unsigned long hmbird_get_dsq_id(struct task_struct *p)
+{
+	struct hmbird_entity *entity = get_hmbird_ts(p);
+
+	if (entity)
+		return (entity->sched_prop & SCHED_PROP_DEADLINE_MASK);
+	else
+		return 0;
+}
+
+static inline void hmbird_set_dsq_sync_ux(struct task_struct *p, int val)
+{
+	struct hmbird_entity *entity = get_hmbird_ts(p);
+
+	if (entity)
+		entity->dsq_sync_ux = val;
+}
+
+static inline int hmbird_get_dsq_sync_ux(struct task_struct *p)
+{
+	struct hmbird_entity *entity = get_hmbird_ts(p);
+
+	if (entity)
+		return entity->dsq_sync_ux;
+	else
+		return 0;
+}
+#endif
 
 extern int can_nice(const struct task_struct *p, const int nice);
 extern int task_curr(const struct task_struct *p);
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
deleted file mode 100755
index b737a00c1373..000000000000
--- a/include/linux/sched/ext.h
+++ /dev/null
@@ -1,647 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#ifndef _LINUX_SCHED_EXT_H
-#define _LINUX_SCHED_EXT_H
-
-#ifdef CONFIG_SCHED_CLASS_EXT
-
-#include <linux/llist.h>
-
-enum scx_consts {
-	SCX_OPS_NAME_LEN	= 128,
-	SCX_EXIT_REASON_LEN	= 128,
-	SCX_EXIT_BT_LEN		= 64,
-	SCX_EXIT_MSG_LEN	= 1024,
-
-	SCX_SLICE_DFL		= 20 * NSEC_PER_MSEC,
-	SCX_SLICE_INF		= U64_MAX,	/* infinite, implies nohz */
-};
-
-/*
- * DSQ (dispatch queue) IDs are 64bit of the format:
- *
- *   Bits: [63] [62 ..  0]
- *         [ B] [   ID   ]
- *
- *    B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs
- *   ID: 63 bit ID
- *
- * Built-in IDs:
- *
- *   Bits: [63] [62] [61..32] [31 ..  0]
- *         [ 1] [ L] [   R  ] [    V   ]
- *
- *    1: 1 for built-in DSQs.
- *    L: 1 for LOCAL_ON DSQ IDs, 0 for others
- *    V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value.
- */
-enum scx_dsq_id_flags {
-	SCX_DSQ_FLAG_BUILTIN	= 1LLU << 63,
-	SCX_DSQ_FLAG_LOCAL_ON	= 1LLU << 62,
-
-	SCX_DSQ_INVALID		= SCX_DSQ_FLAG_BUILTIN | 0,
-	SCX_DSQ_GLOBAL		= SCX_DSQ_FLAG_BUILTIN | 1,
-	SCX_DSQ_LOCAL		= SCX_DSQ_FLAG_BUILTIN | 2,
-	SCX_DSQ_LOCAL_ON	= SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON,
-	SCX_DSQ_LOCAL_CPU_MASK	= 0xffffffffLLU,
-};
-
-enum scx_exit_type {
-	SCX_EXIT_NONE,
-	SCX_EXIT_DONE,
-
-	SCX_EXIT_UNREG = 64,	/* BPF unregistration */
-	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */
-
-	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */
-	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */
-	SCX_EXIT_ERROR_STALL,	/* watchdog detected stalled runnable tasks */
-};
-
-/*
- * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is
- * being disabled.
- */
-struct scx_exit_info {
-	/* %SCX_EXIT_* - broad category of the exit reason */
-	enum scx_exit_type	type;
-	/* textual representation of the above */
-	char			reason[SCX_EXIT_REASON_LEN];
-	/* number of entries in the backtrace */
-	u32			bt_len;
-	/* backtrace if exiting due to an error */
-	unsigned long		bt[SCX_EXIT_BT_LEN];
-	/* extra message */
-	char			msg[SCX_EXIT_MSG_LEN];
-};
-
-/* sched_ext_ops.flags */
-enum scx_ops_flags {
-	/*
-	 * Keep built-in idle tracking even if ops.update_idle() is implemented.
-	 */
-	SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0,
-
-	/*
-	 * By default, if there are no other task to run on the CPU, ext core
-	 * keeps running the current task even after its slice expires. If this
-	 * flag is specified, such tasks are passed to ops.enqueue() with
-	 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.
-	 */
-	SCX_OPS_ENQ_LAST	= 1LLU << 1,
-
-	/*
-	 * An exiting task may schedule after PF_EXITING is set. In such cases,
-	 * bpf_task_from_pid() may not be able to find the task and if the BPF
-	 * scheduler depends on pid lookup for dispatching, the task will be
-	 * lost leading to various issues including RCU grace period stalls.
-	 *
-	 * To mask this problem, by default, unhashed tasks are automatically
-	 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't
-	 * depend on pid lookups and wants to handle these tasks directly, the
-	 * following flag can be used.
-	 */
-	SCX_OPS_ENQ_EXITING	= 1LLU << 2,
-
-	/*
-	 * CPU cgroup knob enable flags
-	 */
-	SCX_OPS_CGROUP_KNOB_WEIGHT = 1LLU << 16,	/* cpu.weight */
-
-	SCX_OPS_ALL_FLAGS	= SCX_OPS_KEEP_BUILTIN_IDLE |
-				  SCX_OPS_ENQ_LAST |
-				  SCX_OPS_ENQ_EXITING |
-				  SCX_OPS_CGROUP_KNOB_WEIGHT,
-};
-
-/* argument container for ops.enable() and friends */
-struct scx_enable_args {
-};
-
-/* argument container for ops->cgroup_init() */
-struct scx_cgroup_init_args {
-	/* the weight of the cgroup [1..10000] */
-	u32			weight;
-};
-
-enum scx_cpu_preempt_reason {
-	/* next task is being scheduled by &sched_class_rt */
-        SCX_CPU_PREEMPT_RT,
-	/* next task is being scheduled by &sched_class_dl */
-        SCX_CPU_PREEMPT_DL,
-	/* next task is being scheduled by &sched_class_stop */
-        SCX_CPU_PREEMPT_STOP,
-	/* unknown reason for SCX being preempted */
-        SCX_CPU_PREEMPT_UNKNOWN,
-};
-
-/*
- * Argument container for ops->cpu_acquire(). Currently empty, but may be
- * expanded in the future.
- */
-struct scx_cpu_acquire_args {};
-
-/* argument container for ops->cpu_release() */
-struct scx_cpu_release_args {
-	/* the reason the CPU was preempted */
-	enum scx_cpu_preempt_reason reason;
-
-	/* the task that's going to be scheduled on the CPU */
-	struct task_struct *task;
-};
-
-/**
- * struct sched_ext_ops - Operation table for BPF scheduler implementation
- *
- * Userland can implement an arbitrary scheduling policy by implementing and
- * loading operations in this table.
- */
-struct sched_ext_ops {
-	/**
-	 * select_cpu - Pick the target CPU for a task which is being woken up
-	 * @p: task being woken up
-	 * @prev_cpu: the cpu @p was on before sleeping
-	 * @wake_flags: SCX_WAKE_*
-	 *
-	 * Decision made here isn't final. @p may be moved to any CPU while it
-	 * is getting dispatched for execution later. However, as @p is not on
-	 * the rq at this point, getting the eventual execution CPU right here
-	 * saves a small bit of overhead down the line.
-	 *
-	 * If an idle CPU is returned, the CPU is kicked and will try to
-	 * dispatch. While an explicit custom mechanism can be added,
-	 * select_cpu() serves as the default way to wake up idle CPUs.
-	 */
-	s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags);
-
-	/**
-	 * enqueue - Enqueue a task on the BPF scheduler
-	 * @p: task being enqueued
-	 * @enq_flags: %SCX_ENQ_*
-	 *
-	 * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch()
-	 * or enqueue on the BPF scheduler. If not directly dispatched, the bpf
-	 * scheduler owns @p and if it fails to dispatch @p, the task will
-	 * stall.
-	 */
-	void (*enqueue)(struct task_struct *p, u64 enq_flags);
-
-	/**
-	 * dequeue - Remove a task from the BPF scheduler
-	 * @p: task being dequeued
-	 * @deq_flags: %SCX_DEQ_*
-	 *
-	 * Remove @p from the BPF scheduler. This is usually called to isolate
-	 * the task while updating its scheduling properties (e.g. priority).
-	 *
-	 * The ext core keeps track of whether the BPF side owns a given task or
-	 * not and can gracefully ignore spurious dispatches from BPF side,
-	 * which makes it safe to not implement this method. However, depending
-	 * on the scheduling logic, this can lead to confusing behaviors - e.g.
-	 * scheduling position not being updated across a priority change.
-	 */
-	void (*dequeue)(struct task_struct *p, u64 deq_flags);
-
-	/**
-	 * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs
-	 * @cpu: CPU to dispatch tasks for
-	 * @prev: previous task being switched out
-	 *
-	 * Called when a CPU's local dsq is empty. The operation should dispatch
-	 * one or more tasks from the BPF scheduler into the DSQs using
-	 * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using
-	 * scx_bpf_consume().
-	 *
-	 * The maximum number of times scx_bpf_dispatch() can be called without
-	 * an intervening scx_bpf_consume() is specified by
-	 * ops.dispatch_max_batch. See the comments on top of the two functions
-	 * for more details.
-	 *
-	 * When not %NULL, @prev is an SCX task with its slice depleted. If
-	 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in
-	 * @prev->scx.flags, it is not enqueued yet and will be enqueued after
-	 * ops.dispatch() returns. To keep executing @prev, return without
-	 * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST.
-	 */
-	void (*dispatch)(s32 cpu, struct task_struct *prev);
-
-	/**
-	 * runnable - A task is becoming runnable on its associated CPU
-	 * @p: task becoming runnable
-	 * @enq_flags: %SCX_ENQ_*
-	 *
-	 * This and the following three functions can be used to track a task's
-	 * execution state transitions. A task becomes ->runnable() on a CPU,
-	 * and then goes through one or more ->running() and ->stopping() pairs
-	 * as it runs on the CPU, and eventually becomes ->quiescent() when it's
-	 * done running on the CPU.
-	 *
-	 * @p is becoming runnable on the CPU because it's
-	 *
-	 * - waking up (%SCX_ENQ_WAKEUP)
-	 * - being moved from another CPU
-	 * - being restored after temporarily taken off the queue for an
-	 *   attribute change.
-	 *
-	 * This and ->enqueue() are related but not coupled. This operation
-	 * notifies @p's state transition and may not be followed by ->enqueue()
-	 * e.g. when @p is being dispatched to a remote CPU. Likewise, a task
-	 * may be ->enqueue()'d without being preceded by this operation e.g.
-	 * after exhausting its slice.
-	 */
-	void (*runnable)(struct task_struct *p, u64 enq_flags);
-
-	/**
-	 * running - A task is starting to run on its associated CPU
-	 * @p: task starting to run
-	 *
-	 * See ->runnable() for explanation on the task state notifiers.
-	 */
-	void (*running)(struct task_struct *p);
-
-	/**
-	 * stopping - A task is stopping execution
-	 * @p: task stopping to run
-	 * @runnable: is task @p still runnable?
-	 *
-	 * See ->runnable() for explanation on the task state notifiers. If
-	 * !@runnable, ->quiescent() will be invoked after this operation
-	 * returns.
-	 */
-	void (*stopping)(struct task_struct *p, bool runnable);
-
-	/**
-	 * quiescent - A task is becoming not runnable on its associated CPU
-	 * @p: task becoming not runnable
-	 * @deq_flags: %SCX_DEQ_*
-	 *
-	 * See ->runnable() for explanation on the task state notifiers.
-	 *
-	 * @p is becoming quiescent on the CPU because it's
-	 *
-	 * - sleeping (%SCX_DEQ_SLEEP)
-	 * - being moved to another CPU
-	 * - being temporarily taken off the queue for an attribute change
-	 *   (%SCX_DEQ_SAVE)
-	 *
-	 * This and ->dequeue() are related but not coupled. This operation
-	 * notifies @p's state transition and may not be preceded by ->dequeue()
-	 * e.g. when @p is being dispatched to a remote CPU.
-	 */
-	void (*quiescent)(struct task_struct *p, u64 deq_flags);
-
-	/**
-	 * yield - Yield CPU
-	 * @from: yielding task
-	 * @to: optional yield target task
-	 *
-	 * If @to is NULL, @from is yielding the CPU to other runnable tasks.
-	 * The BPF scheduler should ensure that other available tasks are
-	 * dispatched before the yielding task. Return value is ignored in this
-	 * case.
-	 *
-	 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf
-	 * scheduler can implement the request, return %true; otherwise, %false.
-	 */
-	bool (*yield)(struct task_struct *from, struct task_struct *to);
-
-	/**
-	 * core_sched_before - Task ordering for core-sched
-	 * @a: task A
-	 * @b: task B
-	 *
-	 * Used by core-sched to determine the ordering between two tasks. See
-	 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on
-	 * core-sched.
-	 *
-	 * Both @a and @b are runnable and may or may not currently be queued on
-	 * the BPF scheduler. Should return %true if @a should run before @b.
-	 * %false if there's no required ordering or @b should run before @a.
-	 *
-	 * If not specified, the default is ordering them according to when they
-	 * became runnable.
-	 */
-	bool (*core_sched_before)(struct task_struct *a,struct task_struct *b);
-
-	/**
-	 * set_weight - Set task weight
-	 * @p: task to set weight for
-	 * @weight: new eight [1..10000]
-	 *
-	 * Update @p's weight to @weight.
-	 */
-	void (*set_weight)(struct task_struct *p, u32 weight);
-
-	/**
-	 * set_cpumask - Set CPU affinity
-	 * @p: task to set CPU affinity for
-	 * @cpumask: cpumask of cpus that @p can run on
-	 *
-	 * Update @p's CPU affinity to @cpumask.
-	 */
-	void (*set_cpumask)(struct task_struct *p, struct cpumask *cpumask);
-
-	/**
-	 * update_idle - Update the idle state of a CPU
-	 * @cpu: CPU to udpate the idle state for
-	 * @idle: whether entering or exiting the idle state
-	 *
-	 * This operation is called when @rq's CPU goes or leaves the idle
-	 * state. By default, implementing this operation disables the built-in
-	 * idle CPU tracking and the following helpers become unavailable:
-	 *
-	 * - scx_bpf_select_cpu_dfl()
-	 * - scx_bpf_test_and_clear_cpu_idle()
-	 * - scx_bpf_pick_idle_cpu()
-	 * - scx_bpf_any_idle_cpu()
-	 *
-	 * The user also must implement ops.select_cpu() as the default
-	 * implementation relies on scx_bpf_select_cpu_dfl().
-	 *
-	 * If you keep the built-in idle tracking, specify the
-	 * %SCX_OPS_KEEP_BUILTIN_IDLE flag.
-	 */
-	void (*update_idle)(s32 cpu, bool idle);
-
-	/**
-	 * cpu_acquire - A CPU is becoming available to the BPF scheduler
-	 * @cpu: The CPU being acquired by the BPF scheduler.
-	 * @args: Acquire arguments, see the struct definition.
-	 *
-	 * A CPU that was previously released from the BPF scheduler is now once
-	 * again under its control.
-	 */
-	void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args);
-
-	/**
-	 * cpu_release - A CPU is taken away from the BPF scheduler
-	 * @cpu: The CPU being released by the BPF scheduler.
-	 * @args: Release arguments, see the struct definition.
-	 *
-	 * The specified CPU is no longer under the control of the BPF
-	 * scheduler. This could be because it was preempted by a higher
-	 * priority sched_class, though there may be other reasons as well. The
-	 * caller should consult @args->reason to determine the cause.
-	 */
-	void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args);
-
-	/**
-	 * cpu_online - A CPU became online
-	 * @cpu: CPU which just came up
-	 *
-	 * @cpu just came online. @cpu doesn't call ops.enqueue() or run tasks
-	 * associated with other CPUs beforehand.
-	 */
-	void (*cpu_online)(s32 cpu);
-
-	/**
-	 * cpu_offline - A CPU is going offline
-	 * @cpu: CPU which is going offline
-	 *
-	 * @cpu is going offline. @cpu doesn't call ops.enqueue() or run tasks
-	 * associated with other CPUs afterwards.
-	 */
-	void (*cpu_offline)(s32 cpu);
-
-	/**
-	 * prep_enable - Prepare to enable BPF scheduling for a task
-	 * @p: task to prepare BPF scheduling for
-	 * @args: enable arguments, see the struct definition
-	 *
-	 * Either we're loading a BPF scheduler or a new task is being forked.
-	 * Prepare BPF scheduling for @p. This operation may block and can be
-	 * used for allocations.
-	 *
-	 * Return 0 for success, -errno for failure. An error return while
-	 * loading will abort loading of the BPF scheduler. During a fork, will
-	 * abort the specific fork.
-	 */
-	s32 (*prep_enable)(struct task_struct *p, struct scx_enable_args *args);
-
-	/**
-	 * enable - Enable BPF scheduling for a task
-	 * @p: task to enable BPF scheduling for
-	 * @args: enable arguments, see the struct definition
-	 *
-	 * Enable @p for BPF scheduling. @p is now in the cgroup specified for
-	 * the preceding prep_enable() and will start running soon.
-	 */
-	void (*enable)(struct task_struct *p, struct scx_enable_args *args);
-
-	/**
-	 * cancel_enable - Cancel prep_enable()
-	 * @p: task being canceled
-	 * @args: enable arguments, see the struct definition
-	 *
-	 * @p was prep_enable()'d but failed before reaching enable(). Undo the
-	 * preparation.
-	 */
-	void (*cancel_enable)(struct task_struct *p,
-			      struct scx_enable_args *args);
-
-	/**
-	 * disable - Disable BPF scheduling for a task
-	 * @p: task to disable BPF scheduling for
-	 *
-	 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded.
-	 * Disable BPF scheduling for @p.
-	 */
-	void (*disable)(struct task_struct *p);
-
-	/*
-	 * All online ops must come before ops.init().
-	 */
-
-	/**
-	 * init - Initialize the BPF scheduler
-	 */
-	s32 (*init)(void);
-
-	/**
-	 * exit - Clean up after the BPF scheduler
-	 * @info: Exit info
-	 */
-	void (*exit)(struct scx_exit_info *info);
-
-	/**
-	 * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch
-	 */
-	u32 dispatch_max_batch;
-
-	/**
-	 * flags - %SCX_OPS_* flags
-	 */
-	u64 flags;
-
-	/**
-	 * timeout_ms - The maximum amount of time, in milliseconds, that a
-	 * runnable task should be able to wait before being scheduled. The
-	 * maximum timeout may not exceed the default timeout of 30 seconds.
-	 *
-	 * Defaults to the maximum allowed timeout value of 30 seconds.
-	 */
-	u32 timeout_ms;
-
-	/**
-	 * name - BPF scheduler's name
-	 *
-	 * Must be a non-zero valid BPF object name including only isalnum(),
-	 * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the
-	 * BPF scheduler is enabled.
-	 */
-	char name[SCX_OPS_NAME_LEN];
-};
-
-/*
- * Dispatch queue (dsq) is a simple FIFO which is used to buffer between the
- * scheduler core and the BPF scheduler. See the documentation for more details.
- */
-struct scx_dispatch_q {
-	raw_spinlock_t		lock;
-	struct list_head	fifo;	/* processed in dispatching order */
-	struct rb_root_cached	priq;	/* processed in p->scx.dsq_vtime order */
-	u32			nr;
-	u64			id;
-	struct llist_node	free_node;
-	struct rcu_head		rcu;
-};
-
-/* scx_entity.flags */
-enum scx_ent_flags {
-	SCX_TASK_QUEUED		= 1 << 0, /* on ext runqueue */
-	SCX_TASK_BAL_KEEP	= 1 << 1, /* balance decided to keep current */
-	SCX_TASK_ENQ_LOCAL	= 1 << 2, /* used by scx_select_cpu_dfl() to set SCX_ENQ_LOCAL */
-
-	SCX_TASK_OPS_PREPPED	= 1 << 8, /* prepared for BPF scheduler enable */
-	SCX_TASK_OPS_ENABLED	= 1 << 9, /* task has BPF scheduler enabled */
-
-	SCX_TASK_WATCHDOG_RESET = 1 << 16, /* task watchdog counter should be reset */
-	SCX_TASK_DEQD_FOR_SLEEP	= 1 << 17, /* last dequeue was for SLEEP */
-
-	SCX_TASK_CURSOR		= 1 << 31, /* iteration cursor, not a task */
-};
-
-/* scx_entity.dsq_flags */
-enum scx_ent_dsq_flags {
-	SCX_TASK_DSQ_ON_PRIQ	= 1 << 0, /* task is queued on the priority queue of a dsq */
-};
-
-/*
- * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from
- * everywhere and the following bits track which kfunc sets are currently
- * allowed for %current. This simple per-task tracking works because SCX ops
- * nest in a limited way. BPF will likely implement a way to allow and disallow
- * kfuncs depending on the calling context which will replace this manual
- * mechanism. See scx_kf_allow().
- */
-enum scx_kf_mask {
-	SCX_KF_UNLOCKED		= 0,	  /* not sleepable, not rq locked */
-	/* all non-sleepables may be nested inside INIT and SLEEPABLE */
-	SCX_KF_INIT		= 1 << 0, /* running ops.init() */
-	SCX_KF_SLEEPABLE	= 1 << 1, /* other sleepable init operations */
-	/* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */
-	SCX_KF_CPU_RELEASE	= 1 << 2, /* ops.cpu_release() */
-	/* ops.dequeue (in REST) may be nested inside DISPATCH */
-	SCX_KF_DISPATCH		= 1 << 3, /* ops.dispatch() */
-	SCX_KF_ENQUEUE		= 1 << 4, /* ops.enqueue() */
-	SCX_KF_REST		= 1 << 5, /* other rq-locked operations */
-
-	__SCX_KF_RQ_LOCKED	= SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH |
-				  SCX_KF_ENQUEUE | SCX_KF_REST,
-	__SCX_KF_TERMINAL	= SCX_KF_ENQUEUE | SCX_KF_REST,
-};
-
-#define RAVG_HIST_SIZE 5
-struct scx_sched_task_stats {
-	u64				mark_start;
-	u64				window_start;
-	u32				sum;
-	u32				sum_history[RAVG_HIST_SIZE];
-	int 			cidx;
-
-	u32				demand;
-	u16				demand_scaled;
-	void 			*sdsq;
-};
-
-
-
-/*
- * The following is embedded in task_struct and contains all fields necessary
- * for a task to be scheduled by SCX.
- */
-struct sched_ext_entity {
-	struct scx_dispatch_q	*dsq;
-	struct {
-		struct list_head	fifo;	/* dispatch order */
-		struct rb_node		priq;	/* p->scx.dsq_vtime order */
-	} dsq_node;
-	struct list_head	watchdog_node;
-	u32			flags;		/* protected by rq lock */
-	u32			dsq_flags;	/* protected by dsq lock */
-	u32			weight;
-	s32			sticky_cpu;
-	s32			holding_cpu;
-	u32			kf_mask;	/* see scx_kf_mask above */
-	struct task_struct	*kf_tasks[2];	/* see SCX_CALL_OP_TASK() */
-	atomic64_t		ops_state;
-	unsigned long		runnable_at;
-#ifdef CONFIG_SCHED_CORE
-	u64			core_sched_at;	/* see scx_prio_less() */
-#endif
-
-	/* BPF scheduler modifiable fields */
-
-	/*
-	 * Runtime budget in nsecs. This is usually set through
-	 * scx_bpf_dispatch() but can also be modified directly by the BPF
-	 * scheduler. Automatically decreased by SCX as the task executes. On
-	 * depletion, a scheduling event is triggered.
-	 *
-	 * This value is cleared to zero if the task is preempted by
-	 * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the
-	 * task ran. Use p->se.sum_exec_runtime instead.
-	 */
-	u64			slice;
-
-	/*
-	 * Used to order tasks when dispatching to the vtime-ordered priority
-	 * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime()
-	 * but can also be modified directly by the BPF scheduler. Modifying it
-	 * while a task is queued on a dsq may mangle the ordering and is not
-	 * recommended.
-	 */
-	u64			dsq_vtime;
-
-	/*
-	 * If set, reject future sched_setscheduler(2) calls updating the policy
-	 * to %SCHED_EXT with -%EACCES.
-	 *
-	 * If set from ops.prep_enable() and the task's policy is already
-	 * %SCHED_EXT, which can happen while the BPF scheduler is being loaded
-	 * or by inhering the parent's policy during fork, the task's policy is
-	 * rejected and forcefully reverted to %SCHED_NORMAL. The number of such
-	 * events are reported through /sys/kernel/debug/sched_ext::nr_rejected.
-	 */
-	bool			disallow;	/* reject switching into SCX */
-
-	/* cold fields */
-	struct list_head	tasks_node;
-	struct task_struct	*task;
-	struct scx_sched_task_stats sts;
-};
-
-void sched_ext_free(struct task_struct *p);
-
-#else	/* !CONFIG_SCHED_CLASS_EXT */
-
-static inline void sched_ext_free(struct task_struct *p) {}
-
-#endif	/* CONFIG_SCHED_CLASS_EXT */
-#endif	/* _LINUX_SCHED_EXT_H */
diff --git a/include/linux/sched/hmbird.h b/include/linux/sched/hmbird.h
index 780a545667e9..3c7861e37ade 100755
--- a/include/linux/sched/hmbird.h
+++ b/include/linux/sched/hmbird.h
@@ -54,6 +54,7 @@
 extern atomic_t non_hmbird_task;
 extern atomic_t __hmbird_ops_enabled;
 #define hmbird_enabled()           atomic_read(&__hmbird_ops_enabled)
+#define MAX_GLOBAL_DSQS (10)
 
 enum hmbird_consts {
 	HMBIRD_SLICE_DFL		= 1 * NSEC_PER_MSEC,
@@ -90,15 +91,13 @@ enum hmbird_dsq_id_flags {
 	HMBIRD_DSQ_LOCAL_CPU_MASK	= 0xffffffffLLU,
 };
 
-enum hmbird_exit_type {
-	HMBIRD_EXIT_NONE,
-	HMBIRD_EXIT_DONE,
-
-	HMBIRD_EXIT_UNREG = 64,	/* BPF unregistration */
-	HMBIRD_EXIT_SYSRQ,		/* requested by 'S' sysrq */
-
-	HMBIRD_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */
+enum hmbird_switch_type {
+	HMBIRD_SWITCH_PROC,
+	HMBIRD_SWITCH_ERR_WDT,
+	HMBIRD_SWITCH_ERR_HB,
+	HMBIRD_SWITCH_ERR_DSQ,
 	HMBIRD_EXIT_ERROR_STALL,	/* watchdog detected stalled runnable tasks */
+	HMBIRD_EXIT_ERROR_HEARTBEAT,	/* heart beat has stopped */
 };
 
 /*
@@ -199,11 +198,176 @@ struct hmbird_entity {
 	int			dsq_sync_ux;
 };
 
+
+/*
+ * All variables use 64bits width,
+ * Avoid parsing problems caused by automatic alignment(with padding) of structures.
+ */
+
+/* NOTING : Must align to 64bits. */
+#define DESC_STR_LEN	(32)
+/* Supporti up to  three-dimensional arrays. */
+#define PARSE_DIMENS	(3)
+struct meta_desc_t {
+	char desc_str[DESC_STR_LEN];
+	u64 len;
+	u64 parse[PARSE_DIMENS];
+};
+
+#define MAX_SWITCHS	(5)
+struct hmbird_switch_t {
+	u64 switch_at;
+	u64 is_success;
+	u64 end_state;
+	u64 switch_reason;
+};
+#define SWITCH_ITEMS	(sizeof(struct hmbird_switch_t) / sizeof(u64))
+
+#define MAX_EXCEPS	(5)
+enum excep_id {
+	NO_CGROUP_L1,
+	MODULE_UNLOAD,
+	ALREADY_ENABLED,
+	ALREADY_DISABLED,
+	INIT_TASK_FAIL,
+	ALLOC_RQSCX_FAIL,
+	DSQ_ID_ERR,
+	CPU_NO_MASK,
+	SCAN_ENTITY_NULL,
+	ITER_RET_NULL,
+	DEQ_DEQING,
+	ENQ_EXIST1,
+	ENQ_EXIST2,
+	TASK_LINKED1,
+	TASK_UNLINKED,
+	TASK_LINKED2,
+	TASK_UNQUED,
+	TASK_UNWATCHED,
+	HMBIRD_OPN,
+	TASK_WATCHED,
+	RQ_NO_RUNNING,
+	EXTRA_FLAGS,
+	HOLDING_CPU1,
+	HOLDING_CPU2,
+	TASK_OPS_PREPPED,
+	TASK_OPS_UNPREPPED,
+	HMBIRD_OPS_ERR,
+	MAX_EXCEP_ID,
+};
+
+struct snap_misc_t {
+	u64 hmbird_enabled;
+	u64 curr_ss;
+	u64 hmbird_ops_enable_state_var;
+	u64 non_ext_task;
+	u64 parctrl_high_ratio;
+	u64 parctrl_low_ratio;
+	u64 parctrl_high_ratio_l;
+	u64 parctrl_low_ratio_l;
+	u64 isoctrl_high_ratio;
+	u64 isoctrl_low_ratio;
+	u64 misfit_ds;
+	u64 partial_enable;
+	u64 iso_free_rescue;
+	u64 isolate_ctrl;
+	u64 snap_jiffies;
+	u64 snap_time;
+};
+#define SNAP_ITEMS	(sizeof(struct snap_misc_t) / sizeof(u64))
+
+struct panic_snapshot_t {
+	/* what time dose the first task of dsq turn in runnable, check starvation */
+	struct meta_desc_t runnable_at_meta;
+	u64 runnable_at[MAX_GLOBAL_DSQS];
+
+	struct meta_desc_t rq_nr_meta;
+	u64 rq_nr[NR_CPUS];
+
+	struct meta_desc_t scxrq_nr_meta;
+	u64 scxrq_nr[NR_CPUS];
+
+	struct meta_desc_t snap_misc_meta;
+	struct snap_misc_t snap_misc;
+};
+
+/*
+ * Do not record info in hot paths unless absolutely necessary,
+ * The impact on performance should be minimized.
+ */
+struct kernel_info_t {
+	struct meta_desc_t sw_rec_meta;
+	struct hmbird_switch_t sw_rec[MAX_SWITCHS];
+
+	struct meta_desc_t sw_idx_meta;
+	u64 sw_idx;
+
+	struct meta_desc_t excep_rec_meta;
+	u64 excep_rec[MAX_EXCEP_ID][MAX_EXCEPS];
+
+	struct meta_desc_t excep_idx_meta;
+	u64 excep_idx[MAX_EXCEP_ID];
+
+	/* snapshot while panic. */
+	struct panic_snapshot_t snap;
+};
+
+struct ko_info_t {};
+
+struct md_meta_t {
+	u64 self_len;
+	u64 unit_size;
+	u64 desc_meta_len;
+	u64 desc_str_len;
+	u64 switches;
+	u64 exceps;
+	u64 global_dsqs;
+	u64 parse_dimens;
+	u64 nr_cpus;
+	u64 real_cpus;
+	u64 nr_meta_desc;
+	u64 dump_real_size;
+};
+
+struct md_info_t {
+	struct md_meta_t meta;
+	struct kernel_info_t kern_dump;
+	struct ko_info_t ko_dump;
+};
+
+static inline void exceps_update(struct md_info_t *rec, int id, unsigned long jiffies)
+{
+	u64 *idx;
+
+	if (!rec)
+		return;
+
+	idx = &rec->kern_dump.excep_idx[id];
+	rec->kern_dump.excep_rec[id][*idx] = jiffies;
+	*idx = ++(*idx) % MAX_EXCEPS;
+}
+
+static inline void sw_update(struct md_info_t *rec, u64 switch_at,
+				u64 is_success, u64 end_state, u64 switch_reason)
+{
+	u64 *idx;
+
+	if (!rec)
+		return;
+
+	idx = &rec->kern_dump.sw_idx;
+	rec->kern_dump.sw_rec[*idx].switch_at = switch_at;
+	rec->kern_dump.sw_rec[*idx].is_success = is_success;
+	rec->kern_dump.sw_rec[*idx].end_state = end_state;
+	rec->kern_dump.sw_rec[*idx].switch_reason = switch_reason;
+	*idx = ++(*idx) % MAX_SWITCHS;
+}
+
 struct hmbird_ops {
 	bool (*scx_enable)(void);
 	bool (*check_non_task)(void);
 	void (*do_sched_yield_before)(long *skip);
 	void (*window_rollover_run_once)(struct rq *rq);
+	void (*hmbird_get_md_info)(unsigned long *vaddr, unsigned long *size);
 };
 
 void hmbird_free(struct task_struct *p);
diff --git a/include/linux/sched/hmbird_proc_val.h b/include/linux/sched/hmbird_proc_val.h
new file mode 100755
index 000000000000..e59708b16ea3
--- /dev/null
+++ b/include/linux/sched/hmbird_proc_val.h
@@ -0,0 +1,22 @@
+#ifndef __HMBORD_PROC_VAL_H__
+#define __HMBORD_PROC_VAL_H__
+
+extern int scx_enable;
+extern int partial_enable;
+extern int cpuctrl_high_ratio;
+extern int cpuctrl_low_ratio;
+extern int slim_stats;
+extern int hmbirdcore_debug;
+extern int slim_for_app;
+extern int misfit_ds;
+extern unsigned int highres_tick_ctrl;
+extern unsigned int highres_tick_ctrl_dbg;
+extern int cpu7_tl;
+extern int slim_walt_ctrl;
+extern int slim_walt_dump;
+extern int slim_walt_policy;
+extern int slim_gov_debug;
+extern int scx_gov_ctrl;
+extern int sched_ravg_window_frame_per_sec;
+
+#endif
\ No newline at end of file
diff --git a/include/linux/sched/hmbird_version.h b/include/linux/sched/hmbird_version.h
new file mode 100755
index 000000000000..b2843881c26f
--- /dev/null
+++ b/include/linux/sched/hmbird_version.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+
+#ifndef _OPLUS_HMBIRD_VERSION_H_
+#define _OPLUS_HMBIRD_VERSION_H_
+#include <linux/of.h>
+#include <linux/string.h>
+#include <linux/printk.h>
+
+enum hmbird_version {
+	HMBIRD_UNINIT,
+	HMBIRD_GKI_VERSION,
+	HMBIRD_OGKI_VERSION,
+	HMBIRD_UNKNOW_VERSION,
+};
+
+static enum hmbird_version hmbird_version_type = HMBIRD_UNINIT;
+
+#define HMBIRD_VERSION_TYPE_CONFIG_PATH "/soc/oplus,hmbird/version_type"
+
+static inline enum hmbird_version get_hmbird_version_type(void)
+{
+	struct device_node *np = NULL;
+	const char *hmbird_version_str = NULL;
+	if (HMBIRD_UNINIT != hmbird_version_type)
+		return hmbird_version_type;
+	np = of_find_node_by_path(HMBIRD_VERSION_TYPE_CONFIG_PATH);
+	if (np) {
+		of_property_read_string(np, "type", &hmbird_version_str);
+		if (NULL != hmbird_version_str) {
+			if (strncmp(hmbird_version_str, "HMBIRD_OGKI", strlen("HMBIRD_OGKI")) == 0) {
+				hmbird_version_type = HMBIRD_OGKI_VERSION;
+				pr_debug("hmbird version use HMBIRD_OGKI_VERSION, set by dtsi");
+			} else if (strncmp(hmbird_version_str, "HMBIRD_GKI", strlen("HMBIRD_GKI")) == 0) {
+				hmbird_version_type = HMBIRD_GKI_VERSION;
+				pr_debug("hmbird version use HMBIRD_GKI_VERSION, set by dtsi");
+			} else {
+				hmbird_version_type = HMBIRD_UNKNOW_VERSION;
+				pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION, set by dtsi");
+			}
+			return hmbird_version_type;
+		}
+	}
+
+	hmbird_version_type = HMBIRD_UNKNOW_VERSION;
+	pr_debug("hmbird version use default HMBIRD_UNKNOW_VERSION");
+	return hmbird_version_type;
+}
+
+#endif /*_OPLUS_HMBIRD_VERSION_H_ */
diff --git a/include/linux/sched/sa_common.h b/include/linux/sched/sa_common.h
new file mode 100755
index 000000000000..6f76d7cfd7bf
--- /dev/null
+++ b/include/linux/sched/sa_common.h
@@ -0,0 +1,797 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020-2022 Oplus. All rights reserved.
+ */
+
+
+#ifndef _OPLUS_SA_COMMON_H_
+#define _OPLUS_SA_COMMON_H_
+
+#include <linux/sched.h>
+#include <linux/list.h>
+#include <linux/types.h>
+#include <linux/atomic.h>
+#include <linux/hashtable.h>
+#include <linux/cgroup-defs.h>
+#if IS_ENABLED(CONFIG_SCHED_WALT)
+#include <linux/sched/walt.h>
+#endif
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/version.h>
+#include <linux/sched/rt.h>
+
+#include "sa_oemdata.h"
+#include "sa_common_struct.h"
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
+#define OPLUS_UX_EEVDF_COMPATIBLE 1
+#endif
+
+#define SA_DEBUG_ON 0
+
+#if (SA_DEBUG_ON >= 1)
+#define DEBUG_BUG_ON(x) BUG_ON(x)
+#define DEBUG_WARN_ON(x) WARN_ON(x)
+#else
+#define DEBUG_BUG_ON(x)
+#define DEBUG_WARN_ON(x)
+#endif
+
+#define ux_err(fmt, ...) \
+		pr_err("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__)
+#define ux_warn(fmt, ...) \
+		pr_warn("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__)
+#define ux_debug(fmt, ...) \
+		pr_info("[sched_assist][%s]"fmt, __func__, ##__VA_ARGS__)
+
+#define UX_MSG_LEN		64
+#define UX_DEPTH_MAX		5
+
+/* define for debug */
+#define DEBUG_SYSTRACE (1 << 0)
+#define DEBUG_FTRACE   (1 << 1)
+#define DEBUG_KMSG     (1 << 2)        /* used for frameboost */
+#define DEBUG_PIPELINE (1 << 3)
+#define DEBUG_DYNAMIC_HZ (1 << 4)
+#define DEBUG_DYNAMIC_PREEMPT (1 << 5)
+#define DEBUG_AMU_INSTRUCTION (1 << 6)
+#define DEBUG_VERBOSE  (1 << 10)       /* used for frameboost */
+#define DEBUG_SET_DSQ_ID (1 << 11) /* used for hmbird */
+
+/* define for sched assist feature */
+#define FEATURE_COMMON (1 << 0)
+#define FEATURE_SPREAD (1 << 1)
+
+#define UX_EXEC_SLICE (4000000U)
+
+/* define for sched assist thread type, keep same as the define in java file */
+#define SA_OPT_CLEAR				(0)
+#define SA_TYPE_LIGHT				(1 << 0)
+#define SA_TYPE_HEAVY				(1 << 1)
+#define SA_TYPE_ANIMATOR			(1 << 2)
+/* SA_TYPE_LISTPICK for camera */
+#define SA_TYPE_LISTPICK			(1 << 3)
+#define SA_TYPE_MQ_VIP				(1 << 4)
+#define SA_OPT_SET					(1 << 7)
+#define SA_OPT_RESET				(1 << 8)
+#define SA_OPT_SET_PRIORITY			(1 << 9)
+
+/* The following ux value only used in kernel */
+#define SA_TYPE_SWIFT				(1 << 14)
+/* clear ux type when dequeue */
+#define SA_TYPE_ONCE				(1 << 15)
+#define SA_TYPE_INHERIT				(1 << 16)
+/* SA_TYPE_COMBINED isn't marked in ux_state, it only exists in return value of function */
+#define SA_TYPE_COMBINED			(1 << 17)
+#define SA_TYPE_URGENT_MASK	(SA_TYPE_LIGHT|SA_TYPE_ANIMATOR|SA_TYPE_SWIFT)
+#define SCHED_ASSIST_UX_MASK	(SA_TYPE_LIGHT|SA_TYPE_HEAVY|SA_TYPE_ANIMATOR|SA_TYPE_LISTPICK|SA_TYPE_SWIFT)
+
+/* load balance operation is performed on the following ux types */
+#define SCHED_ASSIST_LB_UX			(SA_TYPE_SWIFT | SA_TYPE_ANIMATOR | SA_TYPE_LIGHT | SA_TYPE_HEAVY)
+#define POSSIBLE_UX_MASK			(SA_TYPE_SWIFT | \
+									SA_TYPE_LIGHT | \
+									SA_TYPE_HEAVY | \
+									SA_TYPE_ANIMATOR | \
+									SA_TYPE_LISTPICK | \
+									SA_TYPE_ONCE | \
+									SA_TYPE_INHERIT)
+
+#define SCHED_ASSIST_UX_PRIORITY_MASK	(0xFF000000)
+#define SCHED_ASSIST_UX_PRIORITY_SHIFT	24
+
+#define UX_PRIORITY_TOP_APP		0x0A000000
+#define UX_PRIORITY_AUDIO		0x0A000000
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)
+#define UX_PRIORITY_PIPELINE_UI 0x06000000
+#define UX_PRIORITY_PIPELINE    0x05000000
+#endif
+
+/* define for sched assist scene type, keep same as the define in java file */
+#define SA_SCENE_OPT_CLEAR			(0)
+#define SA_LAUNCH					(1 << 0)
+#define SA_SLIDE					(1 << 1)
+#define SA_CAMERA					(1 << 2)
+#define SA_ANIM_START				(1 << 3) /* we care about both launcher and top app */
+#define SA_ANIM						(1 << 4) /* we only care about launcher */
+#define SA_INPUT					(1 << 5)
+#define SA_LAUNCHER_SI				(1 << 6)
+#define SA_SCENE_OPT_SET			(1 << 7)
+
+#define ROOT_UID               0
+#define SYSTEM_UID             1000
+#define FIRST_APPLICATION_UID  10000
+#define LAST_APPLICATION_UID   19999
+#define PER_USER_RANGE         100000
+/* define for clear IM_FLAG
+if im_flag is 70, it should clear im_flag_audio (70 - 64 = 6)
+eg: gerrit patchset "30438485"
+ */
+#define IM_FLAG_CLEAR			64
+
+extern pid_t save_audio_tgid;
+extern pid_t save_top_app_tgid;
+extern unsigned int top_app_type;
+extern int global_lowend_plat_opt;
+
+#ifdef CONFIG_OPLUS_SCHED_HALT_MASK_PRT
+/* This must be the same as the definition of pause_type in walt_halt.c */
+enum oplus_pause_type {
+	OPLUS_HALT,
+	OPLUS_PARTIAL_HALT,
+
+	OPLUS_MAX_PAUSE_TYPE
+};
+extern cpumask_t cur_cpus_halt_mask;
+extern cpumask_t cur_cpus_phalt_mask;
+DECLARE_PER_CPU(int[OPLUS_MAX_PAUSE_TYPE], oplus_cur_pause_client);
+extern void sa_corectl_systrace_c(void);
+#endif /* CONFIG_OPLUS_SCHED_HALT_MASK_PRT */
+
+/* define for boost threshold unit */
+#define BOOST_THRESHOLD_UNIT (51)
+
+
+enum UX_STATE_TYPE {
+	UX_STATE_INVALID = 0,
+	UX_STATE_NONE,
+	UX_STATE_STATIC,
+	UX_STATE_INHERIT,
+	UX_STATE_COMBINED,
+	MAX_UX_STATE_TYPE,
+};
+
+enum INHERIT_UX_TYPE {
+	INHERIT_UX_BINDER = 0,
+	INHERIT_UX_RWSEM,
+	INHERIT_UX_MUTEX,
+	INHERIT_UX_FUTEX,
+	INHERIT_UX_PIFUTEX,
+	INHERIT_UX_MAX,
+};
+
+/*
+ * WANNING:
+ * new flag should be add before MAX_IM_FLAG_TYPE, never change
+ * the value of those existed flag type.
+ */
+enum IM_FLAG_TYPE {
+	IM_FLAG_NONE = 0,
+	IM_FLAG_SURFACEFLINGER,
+	IM_FLAG_HWC,			/* Discarded */
+	IM_FLAG_RENDERENGINE,
+	IM_FLAG_WEBVIEW,
+	IM_FLAG_CAMERA_HAL,
+	IM_FLAG_AUDIO,
+	IM_FLAG_HWBINDER,
+	IM_FLAG_LAUNCHER,
+	IM_FLAG_LAUNCHER_NON_UX_RENDER,
+	IM_FLAG_SS_LOCK_OWNER,
+	IM_FLAG_FORBID_SET_CPU_AFFINITY = 11, /* forbid setting cpu affinity from app */
+	IM_FLAG_SYSTEMSERVER_PID,
+	IM_FLAG_MIDASD,
+	IM_FLAG_AUDIO_CAMERA_HAL, /* audio mode disable camera hal ux */
+	IM_FLAG_AFFINITY_THREAD,
+	IM_FLAG_TPD_SET_CPU_AFFINITY = 16,
+	IM_FLAG_COMPRESS_THREAD = 17, /* compress thread skips locking protect */
+	IM_FLAG_RENDER_THREAD = 18,
+	MAX_IM_FLAG_TYPE,
+};
+
+#define MAX_IM_FLAG_PRIO	MAX_IM_FLAG_TYPE
+
+enum ots_state {
+	OTS_STATE_SET_AFFINITY,
+	OTS_STATE_DDL_ACTIVE,
+	OTS_STATE_DDL_ACTIVE_PREEMPTED,
+	OTS_STATE_MAX,
+};
+
+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN
+DECLARE_PER_CPU(u64, retired_instrs);
+DECLARE_PER_CPU(u64, nvcsw);
+DECLARE_PER_CPU(u64, nivcsw);
+#endif
+
+struct ux_sched_cluster {
+	struct cpumask cpus;
+	unsigned long capacity;
+};
+
+#define OPLUS_NR_CPUS (8)
+#define OPLUS_MAX_CLS (5)
+struct ux_sched_cputopo {
+	int cls_nr;
+	struct ux_sched_cluster sched_cls[OPLUS_NR_CPUS];
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)
+	cpumask_t oplus_cpu_array[2*OPLUS_MAX_CLS][OPLUS_MAX_CLS];
+#endif
+};
+
+struct oplus_rq {
+	/* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */
+	struct rb_root_cached ux_list;
+	/* a tree to track minimum exec vruntime */
+	struct rb_root_cached exec_timeline;
+	/* malloc this spinlock_t instead of built-in to shrank size of oplus_rq */
+	spinlock_t *ux_list_lock;
+	int nr_running;
+	u64 min_vruntime;
+	u64 load_weight;
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL)
+	struct rb_root_cached ddl_root;
+	spinlock_t *ddl_lock;
+#endif
+
+#ifdef CONFIG_LOCKING_PROTECT
+#ifndef CONFIG_LOCKING_LAST_ENTITY
+	struct list_head locking_thread_list;
+	spinlock_t *locking_list_lock;
+	int rq_locking_task;
+#else
+	struct sched_entity *last_entity;
+#endif
+#endif
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)
+	/* for loadbalance */
+	struct oplus_lb lb;
+#endif
+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN
+	struct hrtimer *resched_timer;
+	int cpu;
+#endif
+};
+
+extern int global_debug_enabled;
+extern int global_sched_assist_enabled;
+extern int global_sched_assist_scene;
+extern int global_silver_perf_core;
+extern int global_sched_group_enabled;
+
+struct rq;
+
+#ifdef CONFIG_LOCKING_PROTECT
+struct sched_assist_locking_ops {
+	void (*check_preempt_tick)(struct task_struct *p,
+			unsigned long *ideal_runtime, bool *skip_preempt,
+			unsigned long delta_exec, struct cfs_rq *cfs_rq,
+			struct sched_entity *curr, unsigned int granularity);
+	void (*check_preempt_wakeup)(struct rq *rq, struct task_struct *p, bool *preempt, bool *nopreempt);
+	void (*state_systrace_c)(unsigned int cpu, struct task_struct *p);
+	void (*locking_tick_hit)(struct task_struct *prev, struct task_struct *next);
+#ifndef CONFIG_LOCKING_LAST_ENTITY
+	void (*replace_next_task_fair)(struct rq *rq,
+			struct task_struct **p, struct sched_entity **se, bool *repick, bool simple);
+	void (*enqueue_entity)(struct rq *rq, struct task_struct *p);
+	void (*dequeue_entity)(struct rq *rq, struct task_struct *p);
+#else
+	void (*set_last_entity)(struct task_struct *prev, struct task_struct *next, struct rq *rq);
+	void (*pick_last_entity)(struct rq *rq, struct task_struct **p, struct sched_entity **se, bool *repick, bool simple);
+	void (*clear_last_entity)(struct rq *rq, struct task_struct *p);
+#endif
+	void (*opt_ss_lock_contention)(struct task_struct *p, int old_im, int new_im);
+};
+
+extern struct sched_assist_locking_ops *locking_ops;
+
+
+#define LOCKING_CALL_OP(op, args...) 						\
+do {														\
+	if (locking_ops && locking_ops->op) {					\
+		locking_ops->op(args);								\
+	}														\
+} while (0)
+
+#define LOCKING_CALL_OP_RET(op, args...) 						\
+({																\
+	__typeof__(locking_ops->op(args)) __ret = 0;				\
+	if (locking_ops && locking_ops->op) 							\
+			__ret = locking_ops->op(args);						\
+	__ret; 														\
+})
+
+void register_sched_assist_locking_ops(struct sched_assist_locking_ops *ops);
+#endif
+
+/* attention: before insert .ko, task's list->prev/next is init with 0 */
+static inline bool oplus_list_empty(struct list_head *list)
+{
+	return list_empty(list) || (list->prev == 0 && list->next == 0);
+}
+
+static inline bool oplus_rbtree_empty(struct rb_root_cached *list)
+{
+	return (rb_first_cached(list) == NULL);
+}
+
+/**
+ * Check if there are ux tasks waiting to run on the specified cpu
+ */
+static inline bool orq_has_ux_tasks(struct oplus_rq *orq)
+{
+	return !oplus_rbtree_empty(&orq->ux_list);
+}
+
+/* attention: before insert .ko, task's node->__rb_parent_color is init with 0 */
+static inline bool oplus_rbnode_empty(struct rb_node *node)
+{
+	return RB_EMPTY_NODE(node) || (node->__rb_parent_color == 0);
+}
+
+static inline struct oplus_task_struct *ux_list_first_entry(struct rb_root_cached *list)
+{
+	struct rb_node *leftmost = rb_first_cached(list);
+	if (leftmost == NULL) {
+		return NULL;
+	}
+	return rb_entry(leftmost, struct oplus_task_struct, ux_entry);
+}
+
+static inline struct oplus_task_struct *exec_timeline_first_entry(struct rb_root_cached *tree)
+{
+	struct rb_node *leftmost = rb_first_cached(tree);
+	if (leftmost == NULL) {
+		return NULL;
+	}
+	return rb_entry(leftmost, struct oplus_task_struct, exec_time_node);
+}
+
+typedef bool (*migrate_task_callback_t)(struct task_struct *tsk, int src_cpu, int dst_cpu);
+extern migrate_task_callback_t fbg_migrate_task_callback;
+typedef void (*android_rvh_schedule_handler_t)(struct task_struct *prev,
+	struct task_struct *next, struct rq *rq);
+extern android_rvh_schedule_handler_t fbg_android_rvh_schedule_callback;
+
+extern struct kmem_cache *oplus_task_struct_cachep;
+
+#define ots_to_ts(ots)	(ots->task)
+#define OTS_IDX			0
+#define ORQ_IDX			0
+
+static inline bool test_task_is_fair(struct task_struct *task)
+{
+	DEBUG_BUG_ON(!task);
+
+	/* valid CFS priority is MAX_RT_PRIO..MAX_PRIO-1 */
+	if ((task->prio >= MAX_RT_PRIO) && (task->prio <= MAX_PRIO-1))
+		return true;
+	return false;
+}
+
+static inline bool test_task_is_rt(struct task_struct *task)
+{
+	DEBUG_BUG_ON(!task);
+
+	/* valid RT priority is 0..MAX_RT_PRIO-1 */
+	if (rt_prio(task->prio))
+		return true;
+
+	return false;
+}
+
+static inline struct oplus_task_struct *get_oplus_task_struct(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = NULL;
+
+	/* not Skip idle thread */
+	if (!t)
+		return NULL;
+
+	ots = (struct oplus_task_struct *) READ_ONCE(t->android_oem_data1[OTS_IDX]);
+	if (IS_ERR_OR_NULL(ots))
+		return NULL;
+
+	return ots;
+}
+
+static inline unsigned long oplus_get_im_flag(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return IM_FLAG_NONE;
+
+	return ots->im_flag;
+}
+
+static inline bool is_optimized_audio_thread(struct task_struct *t)
+{
+	unsigned long im_flag;
+
+	im_flag = oplus_get_im_flag(t);
+	if (test_bit(IM_FLAG_AUDIO, &im_flag))
+		return true;
+
+	return false;
+}
+
+static inline void oplus_set_im_flag(struct task_struct *t, int im_flag)
+{
+	struct oplus_task_struct *ots = NULL;
+	ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return;
+
+	set_bit(im_flag, &ots->im_flag);
+}
+
+static inline int get_ots_ux_state(struct oplus_task_struct *ots)
+{
+	int ux_state;
+	int sub_ux_state;
+	int ux_prio;
+	int ret;
+
+	ux_prio = ots->ux_state & SCHED_ASSIST_UX_PRIORITY_MASK;
+	ux_state = ots->ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT);
+	sub_ux_state = ots->sub_ux_state & (SCHED_ASSIST_UX_MASK | SA_TYPE_INHERIT);
+
+	if (sub_ux_state) {
+		ret = ux_state | (sub_ux_state & ~SA_TYPE_INHERIT) | SA_TYPE_COMBINED;
+	} else {
+		ret = ux_state;
+	}
+
+	if (ret & SCHED_ASSIST_UX_MASK) {
+		return ux_prio | ret;
+	}
+
+	return 0;
+}
+
+bool is_multiple_ux(struct oplus_task_struct *ots);
+
+static inline int oplus_get_ux_state(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	return get_ots_ux_state(ots);
+}
+
+static inline int oplus_get_static_ux_state(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	if (ots->ux_state & SA_TYPE_INHERIT) {
+		return 0;
+	}
+	return ots->ux_state;
+}
+
+static inline int oplus_get_sub_ux_state(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	return ots->sub_ux_state;
+}
+
+static inline int oplus_get_inherited_ux_state(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	if (ots->ux_state & SA_TYPE_INHERIT) {
+		return ots->ux_state;
+	}
+	return ots->sub_ux_state;
+}
+
+void oplus_set_ux_state_lock(struct task_struct *t, int ux_state, int inherit_type, bool need_lock_rq);
+
+static inline s64 oplus_get_inherit_ux(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	return atomic64_read(&ots->inherit_ux);
+}
+
+static inline void oplus_set_inherit_ux(struct task_struct *t, s64 inherit_ux)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return;
+
+	atomic64_set(&ots->inherit_ux, inherit_ux);
+}
+
+static inline int oplus_get_ux_depth(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	return ots->ux_depth;
+}
+
+static inline void oplus_set_ux_depth(struct task_struct *t, int ux_depth)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return;
+
+	ots->ux_depth = ux_depth;
+}
+
+static inline u64 oplus_get_enqueue_time(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return 0;
+
+	return ots->enqueue_time;
+}
+
+static inline void oplus_set_enqueue_time(struct task_struct *t, u64 enqueue_time)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return;
+
+	ots->enqueue_time = enqueue_time;
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)
+	/*
+	 * Record the number of task context switches
+	 * and scheduling delays when enqueueing a task.
+	 */
+	ots->snap_pcount = t->sched_info.pcount;
+	ots->snap_run_delay = t->sched_info.run_delay;
+#endif
+}
+
+static inline void oplus_set_inherit_ux_start(struct task_struct *t, u64 start_time)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return;
+
+	ots->inherit_ux_start = t->se.sum_exec_runtime;
+}
+
+static inline void init_task_ux_info(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return;
+
+	RB_CLEAR_NODE(&ots->ux_entry);
+	RB_CLEAR_NODE(&ots->exec_time_node);
+	ots->ux_state = 0;
+	ots->sub_ux_state = 0;
+	atomic64_set(&ots->inherit_ux, 0);
+	ots->ux_depth = 0;
+	ots->enqueue_time = 0;
+	ots->inherit_ux_start = 0;
+	ots->ux_priority = -1;
+	ots->ux_nice = -1;
+	ots->vruntime = 0;
+	ots->preset_vruntime = 0;
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)
+	ots->abnormal_flag = 0;
+#endif
+#ifdef CONFIG_OPLUS_FEATURE_SCHED_SPREAD
+	ots->lb_state = 0;
+	ots->ld_flag = 0;
+#endif
+	ots->exec_calc_runtime = 0;
+	ots->is_update_runtime = 0;
+	ots->target_process = -1;
+	ots->wake_tid = 0;
+	ots->running_start_time = 0;
+	ots->update_running_start_time = false;
+	ots->last_wake_ts = 0;
+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/
+	memset(&ots->lkinfo, 0, sizeof(struct locking_info));
+	INIT_LIST_HEAD(&ots->lkinfo.node);
+/*#endif*/
+	ots->block_start_time = 0;
+#ifdef CONFIG_LOCKING_PROTECT
+	INIT_LIST_HEAD(&ots->locking_entry);
+	ots->locking_start_time = 0;
+	ots->locking_depth = 0;
+#endif
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)
+	/* for loadbalance */
+	ots->snap_pcount = 0;
+	ots->snap_run_delay = 0;
+	plist_node_init(&ots->rtb, MAX_IM_FLAG_PRIO);
+#endif
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)
+	atomic_set(&ots->pipeline_cpu, -1);
+#endif
+
+#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)
+	ots->amu_cycle = 0;
+	ots->amu_instruct = 0;
+#endif
+};
+
+static inline bool test_ux_type(struct task_struct *task, int ux_type)
+{
+	return oplus_get_ux_state(task) & ux_type;
+}
+
+static inline bool is_heavy_ux_task(struct task_struct *t)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(t);
+
+	if (IS_ERR_OR_NULL(ots))
+		return false;
+
+	return get_ots_ux_state(ots) & SA_TYPE_HEAVY;
+}
+
+static inline bool sched_assist_scene(unsigned int scene)
+{
+	if (unlikely(!global_sched_assist_enabled))
+		return false;
+
+	return global_sched_assist_scene & scene;
+}
+
+static inline unsigned long oplus_task_util(struct task_struct *p)
+{
+#if IS_ENABLED(CONFIG_SCHED_WALT)
+	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
+
+	return wts->demand_scaled;
+#else
+	return READ_ONCE(p->se.avg.util_avg);
+#endif
+}
+
+#if IS_ENABLED(CONFIG_SCHED_WALT)
+static inline u32 task_wts_sum(struct task_struct *tsk)
+{
+	struct walt_task_struct *wts = (struct walt_task_struct *) tsk->android_vendor_data1;
+
+	return wts->sum;
+}
+#endif
+
+bool is_min_cluster(int cpu);
+bool is_max_cluster(int cpu);
+bool is_mid_cluster(int cpu);
+bool im_mali(const char *comm);
+bool is_top(struct task_struct *p);
+bool task_is_runnable(struct task_struct *task);
+struct oplus_rq *get_oplus_rq(struct rq *rq);
+
+typedef unsigned long (*kallsyms_lookup_name_t)(const char *name);
+extern kallsyms_lookup_name_t _kallsyms_lookup_name;
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)
+int ux_mask_to_prio(int ux_mask);
+int ux_prio_to_mask(int prio);
+#endif
+
+noinline int tracing_mark_write(const char *buf);
+void hwbinder_systrace_c(unsigned int cpu, int flag);
+void sched_assist_init_oplus_rq(void);
+void queue_ux_thread(struct rq *rq, struct task_struct *p, int enqueue);
+
+void inherit_ux_inc(struct task_struct *task, int type);
+void inherit_ux_sub(struct task_struct *task, int type, int value);
+void set_inherit_ux(struct task_struct *task, int type, int depth, int inherit_val);
+void reset_inherit_ux(struct task_struct *inherit_task, struct task_struct *ux_task, int reset_type);
+void unset_inherit_ux(struct task_struct *task, int type);
+void unset_inherit_ux_value(struct task_struct *task, int type, int value);
+void inc_inherit_ux_refs(struct task_struct *task, int type);
+void clear_all_inherit_type(struct task_struct *p);
+int get_max_inherit_gran(struct task_struct *p);
+
+bool is_heavy_load_top_task(struct task_struct *p);
+bool test_task_is_fair(struct task_struct *task);
+bool test_task_is_rt(struct task_struct *task);
+
+bool test_task_ux(struct task_struct *task);
+bool test_task_ux_depth(int ux_depth);
+bool test_inherit_ux(struct task_struct *task, int type);
+bool test_set_inherit_ux(struct task_struct *task);
+int get_ux_state_type(struct task_struct *task);
+void sched_assist_target_comm(struct task_struct *task, const char *comm);
+unsigned int ux_task_exec_limit(struct task_struct *p);
+
+void update_ux_sched_cputopo(void);
+bool is_task_util_over(struct task_struct *tsk, int threshold);
+bool oplus_task_misfit(struct task_struct *tsk, int cpu);
+ssize_t oplus_show_cpus(const struct cpumask *mask, char *buf);
+void adjust_rt_lowest_mask(struct task_struct *p, struct cpumask *local_cpu_mask, int ret, bool force_adjust);
+bool sa_skip_rt_sync(struct rq *rq, struct task_struct *p, bool *sync);
+void ux_state_systrace_c(unsigned int cpu, struct task_struct *p);
+bool sa_rt_skip_ux_cpu(int cpu);
+int is_vip_mvp(struct task_struct *p);
+
+/* s64 account_ux_runtime(struct rq *rq, struct task_struct *curr); */
+void opt_ss_lock_contention(struct task_struct *p, unsigned long old_im, int new_im);
+
+/* register vender hook in kernel/sched/topology.c */
+void android_vh_build_sched_domains_handler(void *unused, bool has_asym);
+
+/* register vender hook in kernel/sched/rt.c */
+void android_rvh_select_task_rq_rt_handler(void *unused, struct task_struct *p, int prev_cpu, int sd_flag, int wake_flags, int *new_cpu);
+void android_rvh_find_lowest_rq_handler(void *unused, struct task_struct *p, struct cpumask *local_cpu_mask, int ret, int *best_cpu);
+
+/* register vender hook in kernel/sched/core.c */
+void android_rvh_sched_fork_handler(void *unused, struct task_struct *p);
+void android_rvh_after_enqueue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags);
+void android_rvh_dequeue_task_handler(void *unused, struct rq *rq, struct task_struct *p, int flags);
+void android_rvh_schedule_handler(void *unused, struct task_struct *prev, struct task_struct *next, struct rq *rq);
+void android_vh_scheduler_tick_handler(void *unused, struct rq *rq);
+
+void android_vh_account_process_tick_gran_handler(void *unused, struct task_struct *p, struct rq *rq, int user_tick, int *ticks);
+#ifdef CONFIG_OPLUS_FEATURE_TICK_GRAN
+void sa_sched_switch_handler(void *unused, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state);
+#endif
+
+void set_im_flag_with_bit(int im_flag, struct task_struct *task);
+
+/* register vendor hook in kernel/cgroup/cgroup-v1.c */
+void android_vh_cgroup_set_task_handler(void *unused, int ret, struct task_struct *task);
+/* register vendor hook in kernel/signal.c  */
+void android_vh_exit_signal_handler(void *unused, struct task_struct *p);
+void android_rvh_set_cpus_allowed_by_task_handler(void *unused, const struct cpumask *cpu_valid_mask,
+	const struct cpumask *new_mask, struct task_struct *task, unsigned int *dest_cpu);
+void android_rvh_setscheduler_handler(void *unused, struct task_struct *p);
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_BAN_APP_SET_AFFINITY)
+void android_vh_sched_setaffinity_early_handler(void *unused, struct task_struct *task, const struct cpumask *new_mask, bool *skip);
+#endif
+
+#ifdef CONFIG_OPLUS_SCHED_GROUP_OPT
+void android_vh_reweight_entity_handler(void *unused, struct sched_entity *se);
+#endif
+
+#ifdef CONFIG_BLOCKIO_UX_OPT
+int sa_blockio_init(void);
+void sa_blockio_exit(void);
+#endif
+extern struct notifier_block process_exit_notifier_block;
+#endif /* _OPLUS_SA_COMMON_H_ */
diff --git a/include/linux/sched/sa_common_struct.h b/include/linux/sched/sa_common_struct.h
new file mode 100755
index 000000000000..876feff48b36
--- /dev/null
+++ b/include/linux/sched/sa_common_struct.h
@@ -0,0 +1,277 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020-2022 Oplus. All rights reserved.
+ */
+
+/*
+this file is splited from the sa_common.h to adapt the OKI,
+IS_ENABLED is not allowed in here, because the macro will not work in OKI.
+*/
+#ifndef _OPLUS_SA_COMMON_STRUCT_H_
+#define _OPLUS_SA_COMMON_STRUCT_H_
+
+#define MAX_CLUSTER            (4)
+
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/
+/* hot-thread */
+struct task_record {
+#define RECOED_WINSIZE			(1 << 8)
+#define RECOED_WINIDX_MASK		(RECOED_WINSIZE - 1)
+	u8 winidx;
+	u8 count;
+};
+
+#define MAX_TASK_COMM_LEN 256
+struct uid_struct {
+	uid_t uid;
+	u64 uid_total_cycle;
+	u64 uid_total_inst;
+	spinlock_t lock;
+	char leader_comm[TASK_COMM_LEN];
+	char cmdline[MAX_TASK_COMM_LEN];
+};
+
+struct  amu_uid_entry {
+	uid_t uid;
+	struct uid_struct *uid_struct;
+	struct hlist_node node;
+};
+
+/*#endif*/
+
+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/
+struct locking_info {
+	u64 waittime_stamp;
+	u64 holdtime_stamp;
+	/* Used in torture acquire latency statistic.*/
+	u64 acquire_stamp;
+	/*
+	 * mutex or rwsem optimistic spin start time. Because a task
+	 * can't spin both on mutex and rwsem at one time, use one common
+	 * threshold time is OK.
+	 */
+	u64 opt_spin_start_time;
+	struct task_struct *holder;
+	u32 waittype;
+	bool ux_contrib;
+	/*
+	 * Whether task is ux when it's going to be added to mutex or
+	 * rwsem waiter list. It helps us check whether there is ux
+	 * task on mutex or rwsem waiter list. Also, a task can't be
+	 * added to both mutex and rwsem at one time, so use one common
+	 * field is OK.
+	 */
+	bool is_block_ux;
+	u32 kill_flag;
+	/* for cfs enqueue smoothly.*/
+	struct list_head node;
+	struct task_struct *owner;
+	struct list_head lock_head;
+	u64 clear_seq;
+	atomic_t lock_depth;
+};
+/*#endif*/
+
+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/
+#define RAVG_HIST_SIZE 	5
+#define SCX_SLICE_DFL 	(1 * NSEC_PER_MSEC)
+#define SCX_SLICE_INF	U64_MAX
+#define DEFAULT_CGROUP_DL_IDX (8)
+#define EXT_FLAG_RT_CHANGED  	(1 << 0)
+#define EXT_FLAG_CFS_CHANGED 	(1 << 1)
+struct scx_task_stats {
+	u64	mark_start;
+	u64	window_start;
+	u32	sum;
+	u32	sum_history[RAVG_HIST_SIZE];
+	int	cidx;
+	u32	demand;
+	u16	demand_scaled;
+	void	*sdsq;
+};
+/*
+ * The following is embedded in task_struct and contains all fields necessary
+ * for a task to be scheduled by SCX.
+ */
+struct scx_entity {
+	struct scx_dispatch_q	*dsq;
+	struct {
+		struct list_head	fifo;	/* dispatch order */
+		struct rb_node		priq;	/* p->scx.dsq_vtime order */
+	} dsq_node;
+	u32			flags;		/* protected by rq lock */
+	u32			dsq_flags;	/* protected by dsq lock */
+	s32			sticky_cpu;
+	unsigned long		runnable_at;
+	u64			slice;
+	u64			dsq_vtime;
+	int			gdsq_idx;
+	int 			ext_flags;
+	int 			prio_backup;
+	unsigned long		sched_prop;
+	struct scx_task_stats 	sts;
+};
+/*#endif*/
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL)
+
+#define MAX_CPU_FREQ_STATE 32
+#define MAX_CPU_CNT 8
+
+#define STATE_IN_POOL 0
+#define STATE_ACTIVE 1
+
+struct powermodel_freq_task_state {
+	u8 powermodel_enqueued:1;
+	u8 powermodel_index:7;
+};
+
+struct powermodel_cpu_task_state {
+	struct oplus_task_struct *ots;
+	struct powermodel_freq_task_state powermodel_freq_task_states[MAX_CPU_FREQ_STATE];
+	u64 powermodel_last_seq;
+	u64 last_seq;
+	struct list_head node;
+	int state;
+	int pool_id;
+};
+
+#endif
+
+
+/* Please add your own members of task_struct here :) */
+struct oplus_task_struct {
+	/* CONFIG_OPLUS_FEATURE_SCHED_ASSIST */
+	struct rb_node ux_entry;
+	struct rb_node exec_time_node;
+	struct task_struct *task;
+	atomic64_t inherit_ux;
+	u64 enqueue_time;
+	u64 inherit_ux_start;
+	/* u64 sum_exec_baseline; */
+	u64 total_exec;
+	u64 vruntime;
+	u64 preset_vruntime;
+	/* contains ux state
+	 1. if static and inherited ux both exist, static ux stores in ux_state, inherited ux in sub_ux_state.
+	 2. if only static ux exists, static ux stores in ux_state.
+	 2. if only inherited ux exists, inherited ux stores in ux_state */
+	int ux_state;
+	int sub_ux_state;
+	u8 ux_depth;
+	s8 ux_priority;
+	s8 ux_nice;
+	pid_t affinity_pid;
+	pid_t affinity_tgid;
+	unsigned long state;
+	unsigned long im_flag;
+	atomic_t is_vip_mvp;
+
+/* #if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_DDL) */
+	u64 ddl;
+	u64 ddl_active_ts;
+	u64 runnable_ts;
+	struct rb_node ddl_node;
+/* #endif */
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_ABNORMAL_FLAG)*/
+	int abnormal_flag;
+/*#endif*/
+	/* CONFIG_OPLUS_FEATURE_SCHED_SPREAD */
+	int lb_state;
+	int ld_flag:1;
+	/* CONFIG_OPLUS_FEATURE_TASK_LOAD */
+	int is_update_runtime:1;
+	int target_process;
+	u64 wake_tid;
+	u64 running_start_time;
+	bool update_running_start_time;
+	u64 exec_calc_runtime;
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/
+	struct task_record record[MAX_CLUSTER];	/* 2*u64 */
+	u64 block_start_time;
+/*#endif*/
+	/* CONFIG_OPLUS_FEATURE_FRAME_BOOST */
+	struct list_head fbg_list;
+	raw_spinlock_t fbg_list_entry_lock;
+	bool fbg_running; /* task belongs to a group, and in running */
+	u16 fbg_state;
+	s8 preferred_cluster_id;
+	s8 fbg_depth;
+	u64 last_wake_ts;
+	int fbg_cur_group;
+/*#ifdef CONFIG_LOCKING_PROTECT*/
+	unsigned long locking_start_time;
+	unsigned long last_jiffies;
+	struct list_head locking_entry;
+	int locking_depth;
+	int lk_tick_hit;
+/*#endif*/
+/*#if IS_ENABLED(CONFIG_OPLUS_LOCKING_STRATEGY)*/
+	struct locking_info lkinfo;
+/*#endif*/
+/*#if IS_ENABLED(CONFIG_HMBIRD_SCHED_GKI)*/
+	struct scx_entity scx;
+/*#endif*/
+
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_FDLEAK_CHECK)*/
+	u8 fdleak_flag;
+/*#endif*/
+
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/
+	/* for loadbalance */
+	struct plist_node rtb;		/* rt boost task */
+
+	/*
+	 * The following variables are used to calculate the time
+	 * a task spends in the running/runnable state.
+	 */
+	u64 snap_run_delay;
+	unsigned long snap_pcount;
+/*#endif*/
+#if IS_ENABLED(CONFIG_OPLUS_SCHED_TUNE)
+	int stune_idx;
+#endif
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_PIPELINE)*/
+	atomic_t pipeline_cpu;
+/*#endif*/
+
+	/* for oplus secure guard */
+	int sg_flag;
+	int sg_scno;
+	uid_t sg_uid;
+	uid_t sg_euid;
+	gid_t sg_gid;
+	gid_t sg_egid;
+/*#if IS_ENABLED(CONFIG_ARM64_AMU_EXTN) && IS_ENABLED(CONFIG_OPLUS_FEATURE_CPU_JANKINFO)*/
+	struct uid_struct *uid_struct;
+	u64 amu_instruct;
+	u64 amu_cycle;
+/*#endif*/
+	/* for binder ux */
+	int binder_async_ux_enable;
+	bool binder_async_ux_sts;
+	int binder_thread_mode;
+	struct binder_node *binder_thread_node;
+
+/* for powermodel */
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_POWERMODEL)
+	struct powermodel_cpu_task_state *powermodel_cpu_task_states[MAX_CPU_CNT];
+	u64 exec_runtime;
+#endif
+
+#if IS_ENABLED(CONFIG_OPLUS_FEATURE_SCHED_CFBT)
+	int cfbt_cur_group;
+	bool cfbt_running;
+#endif /* CONFIG_OPLUS_FEATURE_SCHED_CFBT */
+} ____cacheline_aligned;
+
+/*#if IS_ENABLED(CONFIG_OPLUS_FEATURE_LOADBALANCE)*/
+#define INVALID_PID						(-1)
+struct oplus_lb {
+	/* used for active_balance to record the running task. */
+	pid_t pid;
+};
+/*#endif*/
+
+#endif /* _OPLUS_SA_COMMON_STRUCT_H_ */
+
diff --git a/include/linux/sched/sa_oemdata.h b/include/linux/sched/sa_oemdata.h
new file mode 100755
index 000000000000..ebbbc754e391
--- /dev/null
+++ b/include/linux/sched/sa_oemdata.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2022 Oplus. All rights reserved.
+ */
+
+
+#ifndef _OPLUS_SA_OEMDATA_H_
+#define _OPLUS_SA_OEMDATA_H_
+
+int sa_oemdata_init(void);
+void sa_oemdata_deinit(void);
+
+#endif /* _OPLUS_SA_OEMDATA_H_ */
diff --git a/include/linux/sched/sched_ext.h b/include/linux/sched/sched_ext.h
new file mode 100755
index 000000000000..9f1afdb8a04b
--- /dev/null
+++ b/include/linux/sched/sched_ext.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+#ifndef _OPLUS_SCHED_EXT_H
+#define _OPLUS_SCHED_EXT_H
+#include "sa_common.h"
+
+#define SCHED_PROP_TOP_THREAD_SHIFT (8)
+#define SCHED_PROP_TOP_THREAD_MASK  (0xf << SCHED_PROP_TOP_THREAD_SHIFT)
+#define SCHED_PROP_DEADLINE_MASK (0xFF) /* deadline for ext sched class */
+#define SCHED_PROP_DEADLINE_LEVEL1 (1)  /* 1ms for user-aware audio tasks */
+#define SCHED_PROP_DEADLINE_LEVEL2 (2)  /* 2ms for user-aware touch tasks */
+#define SCHED_PROP_DEADLINE_LEVEL3 (3)  /* 4ms for user aware dispaly tasks */
+#define SCHED_PROP_DEADLINE_LEVEL4 (4)  /* 6ms */
+#define SCHED_PROP_DEADLINE_LEVEL5 (5)  /* 8ms */
+#define SCHED_PROP_DEADLINE_LEVEL6 (6)  /* 16ms */
+#define SCHED_PROP_DEADLINE_LEVEL7 (7)  /* 32ms */
+#define SCHED_PROP_DEADLINE_LEVEL8 (8)  /* 64ms */
+#define SCHED_PROP_DEADLINE_LEVEL9 (9)  /* 128ms */
+
+static inline int sched_prop_get_top_thread_id(struct task_struct *p)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(p);
+
+	if (!ots) {
+		return -EPERM;
+	}
+
+	return ((ots->scx.sched_prop & SCHED_PROP_TOP_THREAD_MASK) >> SCHED_PROP_TOP_THREAD_SHIFT);
+}
+
+static inline int sched_set_sched_prop(struct task_struct *p, unsigned long sp)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(p);
+
+	if (!ots) {
+		pr_err("scx_sched_ext: sched_set_sched_prop failed! fn=%s\n", __func__);
+		return -EPERM;
+	}
+
+	ots->scx.sched_prop = sp;
+	return 0;
+}
+
+static inline unsigned long sched_get_sched_prop(struct task_struct *p)
+{
+	struct oplus_task_struct *ots = get_oplus_task_struct(p);
+
+	if (!ots) {
+		pr_err("scx_sched_ext: sched_get_sched_prop failed! fn=%s\n", __func__);
+		return (unsigned long)-1;
+	}
+	return ots->scx.sched_prop;
+}
+
+#endif /*_OPLUS_SCHED_EXT_H */
diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h
index 03d35e3eda3c..6a5f0c0d74bd 100644
--- a/include/linux/sched/task.h
+++ b/include/linux/sched/task.h
@@ -61,8 +61,10 @@ extern asmlinkage void schedule_tail(struct task_struct *prev);
 extern void init_idle(struct task_struct *idle, int cpu);
 
 extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
-extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
-extern void sched_cancel_fork(struct task_struct *p);
+extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
+#ifdef CONFIG_HMBIRD_SCHED
+extern void hmbird_cancel_fork(struct task_struct *p);
+#endif
 extern void sched_post_fork(struct task_struct *p);
 extern void sched_dead(struct task_struct *p);
 
diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
index 359a14cc76a4..969716ab4320 100755
--- a/include/uapi/linux/sched.h
+++ b/include/uapi/linux/sched.h
@@ -118,7 +118,7 @@ struct clone_args {
 /* SCHED_ISO: reserved but not implemented yet */
 #define SCHED_IDLE		5
 #define SCHED_DEADLINE		6
-#define SCHED_EXT		7
+#define SCHED_HMBIRD		7
 
 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
 #define SCHED_RESET_ON_FORK     0x40000000
diff --git a/init/Kconfig b/init/Kconfig
index 48c86ee02f26..f0e97bc54978 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1021,10 +1021,6 @@ config RT_GROUP_SCHED
 	  realtime bandwidth for them.
 	  See Documentation/scheduler/sched-rt-group.rst for more information.
 
-config EXT_GROUP_SCHED
-	bool
-	depends on SCHED_CLASS_EXT && CGROUP_SCHED
-	default y
 endif #CGROUP_SCHED
 
 config SCHED_MM_CID
diff --git a/init/init_task.c b/init/init_task.c
index 69a046388995..31ceb0e469f7 100755
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -6,7 +6,6 @@
 #include <linux/sched/sysctl.h>
 #include <linux/sched/rt.h>
 #include <linux/sched/task.h>
-#include <linux/sched/ext.h>
 #include <linux/init.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
@@ -102,9 +101,6 @@ struct task_struct init_task
 #endif
 #ifdef CONFIG_CGROUP_SCHED
 	.sched_task_group = &root_task_group,
-#endif
-#ifdef CONFIG_SLIM_SCHED
-	.scx		= NULL,
 #endif
 	.ptraced	= LIST_HEAD_INIT(init_task.ptraced),
 	.ptrace_entry	= LIST_HEAD_INIT(init_task.ptrace_entry),
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index cf22ef6107b8..b131d5662b48 100755
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -133,12 +133,8 @@ config SCHED_CORE
 	  which is the likely usage by Linux distributions, there should
 	  be no measurable impact on performance.
 
-config SLIM_SCHED
-	bool "slim sched"
-	default n
-config SCHED_CLASS_EXT
-	bool "Extensible Scheduling Class"
-	depends on BPF_SYSCALL && BPF_JIT
+config HMBIRD_SCHED
+	bool "hmbird Scheduling Class(base on ext scheduler)"
 	help
 	  This option enables a new scheduler class sched_ext (SCX), which
 	  allows scheduling policies to be implemented as BPF programs to
@@ -159,3 +155,10 @@ config SCHED_CLASS_EXT
 	  similar to struct sched_class.
 
 	  See Documentation/scheduler/sched-ext.rst for more details.
+
+config DYNAMIC_WALT_SUPPORT
+	bool "Dynamic WALT Support for Extensible Scheduling Class"
+	depends on HMBIRD_SCHED
+	default n
+	help
+	  Enable dynamic WALT support for Extensible Scheduling Class.
diff --git a/kernel/bpf/bpf_struct_ops_types.h b/kernel/bpf/bpf_struct_ops_types.h
index 3618769d853d..5678a9ddf817 100644
--- a/kernel/bpf/bpf_struct_ops_types.h
+++ b/kernel/bpf/bpf_struct_ops_types.h
@@ -9,8 +9,4 @@ BPF_STRUCT_OPS_TYPE(bpf_dummy_ops)
 #include <net/tcp.h>
 BPF_STRUCT_OPS_TYPE(tcp_congestion_ops)
 #endif
-#ifdef CONFIG_SCHED_CLASS_EXT
-#include <linux/sched/ext.h>
-BPF_STRUCT_OPS_TYPE(sched_ext_ops)
-#endif
 #endif
diff --git a/kernel/fork.c b/kernel/fork.c
index f202fd6d20a6..6a393a0d638e 100755
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -23,7 +23,9 @@
 #include <linux/sched/task.h>
 #include <linux/sched/task_stack.h>
 #include <linux/sched/cputime.h>
-#include <linux/sched/ext.h>
+#ifdef CONFIG_HMBIRD_SCHED
+#include <linux/sched/hmbird.h>
+#endif
 #include <linux/seq_file.h>
 #include <linux/rtmutex.h>
 #include <linux/init.h>
@@ -997,7 +999,9 @@ void __put_task_struct(struct task_struct *tsk)
 	WARN_ON(refcount_read(&tsk->usage));
 	WARN_ON(tsk == current);
 
-	sched_ext_free(tsk);
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_free(tsk);
+#endif
 	io_uring_free(tsk);
 	cgroup_free(tsk);
 	task_numa_free(tsk, true);
@@ -2508,7 +2512,11 @@ __latent_entropy struct task_struct *copy_process(
 
 	retval = perf_event_init_task(p, clone_flags);
 	if (retval)
-		goto bad_fork_sched_cancel_fork;
+#ifdef CONFIG_HMBIRD_SCHED
+		goto bad_fork_hmbird_cancel_fork;
+#else
+		goto bad_fork_cleanup_policy;
+#endif
 	retval = audit_alloc(p);
 	if (retval)
 		goto bad_fork_cleanup_perf;
@@ -2640,9 +2648,7 @@ __latent_entropy struct task_struct *copy_process(
 	 * cgroup specific, it unconditionally needs to place the task on a
 	 * runqueue.
 	 */
-	retval = sched_cgroup_fork(p, args);
-	if (retval)
-		goto bad_fork_cancel_cgroup;
+	sched_cgroup_fork(p, args);
 
 	/*
 	 * From this point on we must avoid any synchronous user-space
@@ -2688,13 +2694,13 @@ __latent_entropy struct task_struct *copy_process(
 	/* Don't start children in a dying pid namespace */
 	if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
 		retval = -ENOMEM;
-		goto bad_fork_core_free;
+		goto bad_fork_cancel_cgroup;
 	}
 
 	/* Let kill terminate clone/fork in the middle */
 	if (fatal_signal_pending(current)) {
 		retval = -EINTR;
-		goto bad_fork_core_free;
+		goto bad_fork_cancel_cgroup;
 	}
 
 	/* No more failure paths after this point. */
@@ -2771,11 +2777,10 @@ __latent_entropy struct task_struct *copy_process(
 
 	return p;
 
-bad_fork_core_free:
+bad_fork_cancel_cgroup:
 	sched_core_free(p);
 	spin_unlock(&current->sighand->siglock);
 	write_unlock_irq(&tasklist_lock);
-bad_fork_cancel_cgroup:
 	cgroup_cancel_fork(p, args);
 bad_fork_put_pidfd:
 	if (clone_flags & CLONE_PIDFD) {
@@ -2814,8 +2819,10 @@ __latent_entropy struct task_struct *copy_process(
 	audit_free(p);
 bad_fork_cleanup_perf:
 	perf_event_free_task(p);
-bad_fork_sched_cancel_fork:
-	sched_cancel_fork(p);
+#ifdef CONFIG_HMBIRD_SCHED
+bad_fork_hmbird_cancel_fork:
+	hmbird_cancel_fork(p);
+#endif
 bad_fork_cleanup_policy:
 	lockdep_free_task(p);
 #ifdef CONFIG_NUMA
diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c
index 005025f55bea..c091539a0f60 100755
--- a/kernel/sched/build_policy.c
+++ b/kernel/sched/build_policy.c
@@ -28,8 +28,10 @@
 #include <linux/suspend.h>
 #include <linux/tsacct_kern.h>
 #include <linux/vtime.h>
+#ifdef CONFIG_HMBIRD_SCHED
 #include <linux/sysrq.h>
 #include <linux/percpu-rwsem.h>
+#endif
 
 #include <uapi/linux/sched/types.h>
 
@@ -54,6 +56,10 @@
 #include "cputime.c"
 #include "deadline.c"
 
-#ifdef CONFIG_SCHED_CLASS_EXT
-# include "ext.c"
+#ifdef CONFIG_HMBIRD_SCHED
+#include "hmbird/hmbird_util_track.c"
+#include "hmbird/hmbird_sched_proc.c"
+#include "hmbird/hmbird_shadow_tick.c"
+#include "hmbird/hmbird.c"
+#include "hmbird/hmbird_misc.c"
 #endif
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 85ca05ced366..a258adcea40c 100755
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -96,6 +96,12 @@
 #include "../../io_uring/io-wq.h"
 #include "../smpboot.h"
 
+#ifdef CONFIG_HMBIRD_SCHED
+#include "hmbird/slim.h"
+#include "hmbird/hmbird_shadow_tick.h"
+#include "hmbird.h"
+#endif
+
 #include <trace/hooks/sched.h>
 #include <trace/hooks/cgroup.h>
 #include <trace/hooks/dtask.h>
@@ -170,7 +176,6 @@ __read_mostly int scheduler_running;
 
 DEFINE_STATIC_KEY_FALSE(__sched_core_enabled);
 
-
 /* kernel prio, less is more */
 static inline int __task_prio(const struct task_struct *p)
 {
@@ -183,8 +188,8 @@ static inline int __task_prio(const struct task_struct *p)
 	if (p->sched_class == &idle_sched_class)
 		return MAX_RT_PRIO + NICE_WIDTH; /* 140 */
 
-#ifdef CONFIG_SCHED_CLASS_EXT
-	if (p->sched_class == &ext_sched_class)
+#ifdef CONFIG_HMBIRD_SCHED
+	if (p->sched_class == &hmbird_sched_class)
 		return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */
 #endif
 
@@ -217,9 +222,9 @@ static inline bool prio_less(const struct task_struct *a,
 	if (pa == MAX_RT_PRIO + MAX_NICE)	/* fair */
 		return cfs_prio_less(a, b, in_fi);
 
-#ifdef CONFIG_SCHED_CLASS_EXT
+#ifdef CONFIG_HMBIRD_SCHED
 	if (pa == MAX_RT_PRIO + MAX_NICE + 1)	/* ext */
-		return scx_prio_less(a, b, in_fi);
+		return hmbird_prio_less(a, b, in_fi);
 #endif
 
 	return false;
@@ -1276,17 +1281,26 @@ bool sched_can_stop_tick(struct rq *rq)
 	if (fifo_nr_running)
 		return true;
 
+#ifdef CONFIG_HMBIRD_SCHED
 	/*
-	 * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks
+	 * If there are no DL,RR/FIFO tasks, there must only be CFS or HMBIRD tasks
 	 * left. For CFS, if there's more than one we need the tick for
-	 * involuntary preemption. For SCX, ask.
+	 * involuntary preemption. For HMBIRD, ask.
 	 */
-	if (!scx_switched_all() && rq->nr_running > 1)
+	if (!hmbird_enabled() && rq->nr_running > 1)
 		return false;
 
-	if (scx_enabled() && !scx_can_stop_tick(rq))
+	if (hmbird_enabled() && !hmbird_can_stop_tick(rq))
 		return false;
-
+#else
+	/*
+	 * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
+	 * if there's more than one we need the tick for involuntary
+	 * preemption.
+	 */
+	if (rq->nr_running > 1)
+		return false;
+#endif
 	/*
 	 * If there is one task and it has CFS runtime bandwidth constraints
 	 * and it's on the cpu now we don't want to stop the tick.
@@ -2211,10 +2225,11 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 }
 EXPORT_SYMBOL_GPL(deactivate_task);
 
-struct sched_change_guard
-sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags)
+#ifdef CONFIG_HMBIRD_SCHED
+struct hmbird_sched_change_guard
+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags)
 {
-	struct sched_change_guard cg = {
+	struct hmbird_sched_change_guard cg = {
 		.rq = rq,
 		.p = p,
 		.queued = task_on_rq_queued(p),
@@ -2236,7 +2251,7 @@ sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags)
 	return cg;
 }
 
-void sched_change_guard_fini(struct sched_change_guard *cg, int flags)
+void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags)
 {
 	if (cg->queued)
 		enqueue_task(cg->rq, cg->p, flags | ENQUEUE_NOCLOCK);
@@ -2244,6 +2259,7 @@ void sched_change_guard_fini(struct sched_change_guard *cg, int flags)
 		set_next_task(cg->rq, cg->p);
 	cg->done = true;
 }
+#endif
 
 static inline int __normal_prio(int policy, int rt_prio, int nice)
 {
@@ -2309,9 +2325,9 @@ inline int task_curr(const struct task_struct *p)
  * this means any call to check_class_changed() must be followed by a call to
  * balance_callback().
  */
-void check_class_changed(struct rq *rq, struct task_struct *p,
-			 const struct sched_class *prev_class,
-			 int oldprio)
+static inline void check_class_changed(struct rq *rq, struct task_struct *p,
+				       const struct sched_class *prev_class,
+				       int oldprio)
 {
 	if (prev_class != p->sched_class) {
 		if (prev_class->switched_from)
@@ -2874,6 +2890,7 @@ static void
 __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
 {
 	struct rq *rq = task_rq(p);
+	bool queued, running;
 
 	/*
 	 * This here violates the locking rules for affinity, since we're only
@@ -2892,9 +2909,26 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
 	else
 		lockdep_assert_held(&p->pi_lock);
 
-	SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) {
-		p->sched_class->set_cpus_allowed(p, ctx);
+	queued = task_on_rq_queued(p);
+	running = task_current(rq, p);
+
+	if (queued) {
+		/*
+		 * Because __kthread_bind() calls this on blocked tasks without
+		 * holding rq->lock.
+		 */
+		lockdep_assert_rq_held(rq);
+		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
 	}
+	if (running)
+		put_prev_task(rq, p);
+
+	p->sched_class->set_cpus_allowed(p, ctx);
+
+	if (queued)
+		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
+	if (running)
+		set_next_task(rq, p);
 }
 
 /*
@@ -3761,7 +3795,11 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags)
 	 * [ this allows ->select_task() to simply return task_cpu(p) and
 	 *   not worry about this generic constraint ]
 	 */
+#ifdef CONFIG_HMBIRD_SCHED
+	if (unlikely(!is_cpu_allowed(p, cpu)) && (p->sched_class != &hmbird_sched_class))
+#else
 	if (unlikely(!is_cpu_allowed(p, cpu)))
+#endif
 		cpu = select_fallback_rq(task_cpu(p), p);
 
 	return cpu;
@@ -4879,8 +4917,9 @@ late_initcall(sched_core_sysctl_init);
  */
 int sched_fork(unsigned long clone_flags, struct task_struct *p)
 {
-
+#ifdef CONFIG_HMBIRD_SCHED
 	int ret;
+#endif
 
 	trace_android_rvh_sched_fork(p);
 
@@ -4921,21 +4960,29 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
 		p->sched_reset_on_fork = 0;
 	}
 
-	scx_pre_fork(p);
+#ifdef CONFIG_HMBIRD_SCHED
+	ret = hmbird_pre_fork(p);
+	if (ret)
+		goto out_cancel;
 
 	if (dl_prio(p->prio)) {
 		ret = -EAGAIN;
 		goto out_cancel;
-#ifdef CONFIG_SCHED_CLASS_EXT
-	} else if (task_on_scx(p)) {
-		p->sched_class = &ext_sched_class;
-#endif
+	} else if (task_on_hmbird(p)) {
+		p->sched_class = &hmbird_sched_class;
 	} else if (rt_prio(p->prio)) {
 		p->sched_class = &rt_sched_class;
 	} else {
 		p->sched_class = &fair_sched_class;
 	}
-
+#else
+	if (dl_prio(p->prio))
+		return -EAGAIN;
+	else if (rt_prio(p->prio))
+		p->sched_class = &rt_sched_class;
+	else
+		p->sched_class = &fair_sched_class;
+#endif
 	init_entity_runnable_average(&p->se);
 	trace_android_rvh_finish_prio_fork(p);
 
@@ -4954,12 +5001,14 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
 #endif
 	return 0;
 
+#ifdef CONFIG_HMBIRD_SCHED
 out_cancel:
-	scx_cancel_fork(p);
+	hmbird_cancel_fork(p);
 	return ret;
+#endif
 }
 
-int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
+void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
 {
 	unsigned long flags;
 
@@ -4986,20 +5035,17 @@ int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
 	if (p->sched_class->task_fork)
 		p->sched_class->task_fork(p);
 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-
-	return scx_fork(p);
-}
-
-void sched_cancel_fork(struct task_struct *p)
-{
-	scx_cancel_fork(p);
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_fork(p);
+#endif
 }
 
 void sched_post_fork(struct task_struct *p)
 {
 	uclamp_post_fork(p);
-
-	scx_post_fork(p);
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_post_fork(p);
+#endif
 }
 
 unsigned long to_ratio(u64 period, u64 runtime)
@@ -5864,20 +5910,28 @@ void scheduler_tick(void)
 	if (sched_feat(LATENCY_WARN) && resched_latency)
 		resched_latency_warn(cpu, resched_latency);
 
-	scx_notify_sched_tick();
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_notify_sched_tick();
+#endif
 	perf_event_task_tick();
 
 	if (curr->flags & PF_WQ_WORKER)
 		wq_worker_tick(curr);
 
 #ifdef CONFIG_SMP
-	if (!scx_switched_all()) {
+#ifdef CONFIG_HMBIRD_SCHED
+	if (!hmbird_enabled()) {
+#endif
 		rq->idle_balance = idle_cpu(cpu);
 		trigger_load_balance(rq);
+#ifdef CONFIG_HMBIRD_SCHED
 	}
+#endif
 #endif
 	trace_android_vh_scheduler_tick(rq);
-
+#ifdef CONFIG_HMBIRD_SCHED
+	scheduler_tick_handler(NULL, NULL);
+#endif
 }
 
 #ifdef CONFIG_NO_HZ_FULL
@@ -6179,7 +6233,11 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,
 	 * We can terminate the balance pass as soon as we know there is
 	 * a runnable task of @class priority or higher.
 	 */
+#ifdef CONFIG_HMBIRD_SCHED
 	for_balance_class_range(class, prev->sched_class, &idle_sched_class) {
+#else
+	for_class_range(class, prev->sched_class, &idle_sched_class) {
+#endif
 		if (class->balance(rq, prev, rf))
 			break;
 	}
@@ -6197,9 +6255,10 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 	const struct sched_class *class;
 	struct task_struct *p;
 
-	if (scx_enabled())
+#ifdef CONFIG_HMBIRD_SCHED
+	if (hmbird_enabled())
 		goto restart;
-
+#endif
 	/*
 	 * Optimization: we know that if all tasks are in the fair class we can
 	 * call that function directly, but only if the @prev task wasn't of a
@@ -6225,13 +6284,21 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 restart:
 	put_prev_task_balance(rq, prev, rf);
 
+#ifdef CONFIG_HMBIRD_SCHED
 	for_each_active_class(class) {
 		p = class->pick_next_task(rq);
 		if (p) {
-			scx_notify_pick_next_task(rq, p, class);
+			hmbird_notify_pick_next_task(rq, p, class);
 			return p;
 		}
 	}
+#else
+	for_each_class(class) {
+		p = class->pick_next_task(rq);
+		if (p)
+			return p;
+	}
+#endif
 
 	BUG(); /* The idle class should always have a runnable task. */
 }
@@ -6260,7 +6327,11 @@ static inline struct task_struct *pick_task(struct rq *rq)
 	const struct sched_class *class;
 	struct task_struct *p;
 
+#ifdef CONFIG_HMBIRD_SCHED
 	for_each_active_class(class) {
+#else
+	for_each_class(class) {
+#endif
 		p = class->pick_task(rq);
 		if (p)
 			return p;
@@ -6904,6 +6975,9 @@ static void __sched notrace __schedule(unsigned int sched_mode)
 		psi_sched_switch(prev, next, !task_on_rq_queued(prev));
 
 		trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);
+#ifdef CONFIG_HMBIRD_SCHED
+		sched_switch_handler(NULL, sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);
+#endif
 
 		/* Also unlocks the rq: */
 		rq = context_switch(rq, prev, next, &rf);
@@ -7232,24 +7306,31 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag
 }
 EXPORT_SYMBOL(default_wake_function);
 
-void __setscheduler_prio(struct task_struct *p, int prio)
+static void __setscheduler_prio(struct task_struct *p, int prio)
 {
-	/*
-	 * After switching all rt and fair class to ext,
-	 * stop class is switched to ext class too. Just
-	 * skip it. */
+#ifdef CONFIG_HMBIRD_SCHED
+	bool on_hmbird = task_on_hmbird(p);
+
 	if (p->sched_class == &stop_sched_class)
-		;/* do nothing */
+		;
 	else if (dl_prio(prio))
 		p->sched_class = &dl_sched_class;
-#ifdef CONFIG_SCHED_CLASS_EXT
-	else if (task_on_scx(p))
-		p->sched_class = &ext_sched_class;
-#endif
+	else if (rt_prio(prio) && on_hmbird)
+		p->sched_class = &hmbird_sched_class;
+	else if (rt_prio(prio))
+		p->sched_class = &rt_sched_class;
+	else if (on_hmbird)
+		p->sched_class = &hmbird_sched_class;
+	else
+		p->sched_class = &fair_sched_class;
+#else
+	if (dl_prio(prio))
+		p->sched_class = &dl_sched_class;
 	else if (rt_prio(prio))
 		p->sched_class = &rt_sched_class;
 	else
 		p->sched_class = &fair_sched_class;
+#endif
 
 	p->prio = prio;
 }
@@ -7284,7 +7365,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio)
  */
 void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
 {
-	int prio, oldprio, queue_flag =
+	int prio, oldprio, queued, running, queue_flag =
 		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
 	const struct sched_class *prev_class;
 	struct rq_flags rf;
@@ -7347,42 +7428,52 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
 		queue_flag &= ~DEQUEUE_MOVE;
 
 	prev_class = p->sched_class;
-	SCHED_CHANGE_BLOCK(rq, p, queue_flag) {
-		/*
-		 * Boosting condition are:
-		 * 1. -rt task is running and holds mutex A
-		 *      --> -dl task blocks on mutex A
-		 *
-		 * 2. -dl task is running and holds mutex A
-		 *      --> -dl task blocks on mutex A and could preempt the
-		 *          running task
-		 */
-		if (dl_prio(prio)) {
-			if (!dl_prio(p->normal_prio) ||
-			    (pi_task && dl_prio(pi_task->prio) &&
-			     dl_entity_preempt(&pi_task->dl, &p->dl))) {
-				p->dl.pi_se = pi_task->dl.pi_se;
-				queue_flag |= ENQUEUE_REPLENISH;
-			} else {
-				p->dl.pi_se = &p->dl;
-			}
-		} else if (rt_prio(prio)) {
-			if (dl_prio(oldprio))
-				p->dl.pi_se = &p->dl;
-			if (oldprio < prio)
-				queue_flag |= ENQUEUE_HEAD;
+	queued = task_on_rq_queued(p);
+	running = task_current(rq, p);
+	if (queued)
+		dequeue_task(rq, p, queue_flag);
+	if (running)
+		put_prev_task(rq, p);
+
+	/*
+	 * Boosting condition are:
+	 * 1. -rt task is running and holds mutex A
+	 *      --> -dl task blocks on mutex A
+	 *
+	 * 2. -dl task is running and holds mutex A
+	 *      --> -dl task blocks on mutex A and could preempt the
+	 *          running task
+	 */
+	if (dl_prio(prio)) {
+		if (!dl_prio(p->normal_prio) ||
+		    (pi_task && dl_prio(pi_task->prio) &&
+		     dl_entity_preempt(&pi_task->dl, &p->dl))) {
+			p->dl.pi_se = pi_task->dl.pi_se;
+			queue_flag |= ENQUEUE_REPLENISH;
 		} else {
-			if (dl_prio(oldprio))
-				p->dl.pi_se = &p->dl;
-			else if (rt_prio(oldprio))
-				p->rt.timeout = 0;
-			else if (!task_has_idle_policy(p))
-				reweight_task(p, prio - MAX_RT_PRIO);
+			p->dl.pi_se = &p->dl;
 		}
-
-		__setscheduler_prio(p, prio);
+	} else if (rt_prio(prio)) {
+		if (dl_prio(oldprio))
+			p->dl.pi_se = &p->dl;
+		if (oldprio < prio)
+			queue_flag |= ENQUEUE_HEAD;
+	} else {
+		if (dl_prio(oldprio))
+			p->dl.pi_se = &p->dl;
+		else if (rt_prio(oldprio))
+			p->rt.timeout = 0;
+		else if (!task_has_idle_policy(p))
+			reweight_task(p, prio - MAX_RT_PRIO);
 	}
 
+	__setscheduler_prio(p, prio);
+
+	if (queued)
+		enqueue_task(rq, p, queue_flag);
+	if (running)
+		set_next_task(rq, p);
+
 	check_class_changed(rq, p, prev_class, oldprio);
 out_unlock:
 	/* Avoid rq from going away on us: */
@@ -7403,7 +7494,7 @@ static inline int rt_effective_prio(struct task_struct *p, int prio)
 
 void set_user_nice(struct task_struct *p, long nice)
 {
-	bool allowed = false;
+	bool queued, running, allowed = false;
 	int old_prio;
 	struct rq_flags rf;
 	struct rq *rq;
@@ -7432,13 +7523,22 @@ void set_user_nice(struct task_struct *p, long nice)
 		p->static_prio = NICE_TO_PRIO(nice);
 		goto out_unlock;
 	}
+	queued = task_on_rq_queued(p);
+	running = task_current(rq, p);
+	if (queued)
+		dequeue_task(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK);
+	if (running)
+		put_prev_task(rq, p);
 
-	SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) {
-		p->static_prio = NICE_TO_PRIO(nice);
-		set_load_weight(p, true);
-		old_prio = p->prio;
-		p->prio = effective_prio(p);
-	}
+	p->static_prio = NICE_TO_PRIO(nice);
+	set_load_weight(p, true);
+	old_prio = p->prio;
+	p->prio = effective_prio(p);
+
+	if (queued)
+		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
+	if (running)
+		set_next_task(rq, p);
 
 	/*
 	 * If the task increased its priority or is running and
@@ -7847,7 +7947,7 @@ static int __sched_setscheduler(struct task_struct *p,
 				bool user, bool pi)
 {
 	int oldpolicy = -1, policy = attr->sched_policy;
-	int retval, oldprio, newprio;
+	int retval, oldprio, newprio, queued, running;
 	const struct sched_class *prev_class;
 	struct balance_callback *head;
 	struct rq_flags rf;
@@ -7855,6 +7955,9 @@ static int __sched_setscheduler(struct task_struct *p,
 	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
 	struct rq *rq;
 	bool cpuset_locked = false;
+#ifdef CONFIG_HMBIRD_SCHED
+	unsigned long flags;
+#endif
 
 
 	/* The pi code expects interrupts enabled */
@@ -7921,7 +8024,9 @@ static int __sched_setscheduler(struct task_struct *p,
 	 * To be able to change p->policy safely, the appropriate
 	 * runqueue lock must be held.
 	 */
-
+#ifdef CONFIG_HMBIRD_SCHED
+	spin_lock_irqsave(&hmbird_tasks_lock, flags);
+#endif
 	rq = task_rq_lock(p, &rf);
 	update_rq_clock(rq);
 
@@ -7933,10 +8038,11 @@ static int __sched_setscheduler(struct task_struct *p,
 		goto unlock;
 	}
 
-	retval = scx_check_setscheduler(p, policy);
+#ifdef CONFIG_HMBIRD_SCHED
+	retval = hmbird_check_setscheduler(p, policy);
 	if (retval)
 		goto unlock;
-
+#endif
 	/*
 	 * If not changing anything there's no need to proceed further,
 	 * but store a possible modification of reset_on_fork.
@@ -7993,7 +8099,9 @@ static int __sched_setscheduler(struct task_struct *p,
 	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
 		policy = oldpolicy = -1;
 		task_rq_unlock(rq, p, &rf);
-
+#ifdef CONFIG_HMBIRD_SCHED
+		spin_unlock_irqrestore(&hmbird_tasks_lock, flags);
+#endif
 		if (cpuset_locked)
 			cpuset_unlock();
 		goto recheck;
@@ -8026,16 +8134,23 @@ static int __sched_setscheduler(struct task_struct *p,
 			queue_flags &= ~DEQUEUE_MOVE;
 	}
 
-	SCHED_CHANGE_BLOCK(rq, p, queue_flags) {
-		prev_class = p->sched_class;
+	queued = task_on_rq_queued(p);
+	running = task_current(rq, p);
+	if (queued)
+		dequeue_task(rq, p, queue_flags);
+	if (running)
+		put_prev_task(rq, p);
+
+	prev_class = p->sched_class;
 
-		if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) {
-			__setscheduler_params(p, attr);
-			__setscheduler_prio(p, newprio);
-			trace_android_rvh_setscheduler(p);
-		}
-		__setscheduler_uclamp(p, attr);
+	if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) {
+		__setscheduler_params(p, attr);
+		__setscheduler_prio(p, newprio);
+		trace_android_rvh_setscheduler(p);
+	}
+	__setscheduler_uclamp(p, attr);
 
+	if (queued) {
 		/*
 		 * We enqueue to tail when the priority of a task is
 		 * increased (user space view).
@@ -8043,7 +8158,10 @@ static int __sched_setscheduler(struct task_struct *p,
 		if (oldprio < p->prio)
 			queue_flags |= ENQUEUE_HEAD;
 
+		enqueue_task(rq, p, queue_flags);
 	}
+	if (running)
+		set_next_task(rq, p);
 
 	check_class_changed(rq, p, prev_class, oldprio);
 
@@ -8051,7 +8169,9 @@ static int __sched_setscheduler(struct task_struct *p,
 	preempt_disable();
 	head = splice_balance_callbacks(rq);
 	task_rq_unlock(rq, p, &rf);
-
+#ifdef CONFIG_HMBIRD_SCHED
+	spin_unlock_irqrestore(&hmbird_tasks_lock, flags);
+#endif
 	if (pi) {
 		if (cpuset_locked)
 			cpuset_unlock();
@@ -8066,7 +8186,9 @@ static int __sched_setscheduler(struct task_struct *p,
 
 unlock:
 	task_rq_unlock(rq, p, &rf);
-
+#ifdef CONFIG_HMBIRD_SCHED
+	spin_unlock_irqrestore(&hmbird_tasks_lock, flags);
+#endif
 	if (cpuset_locked)
 		cpuset_unlock();
 	return retval;
@@ -9288,7 +9410,9 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
 	case SCHED_NORMAL:
 	case SCHED_BATCH:
 	case SCHED_IDLE:
-	case SCHED_EXT:
+#ifdef CONFIG_HMBIRD_SCHED
+	case SCHED_HMBIRD:
+#endif
 		ret = 0;
 		break;
 	}
@@ -9316,7 +9440,9 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
 	case SCHED_NORMAL:
 	case SCHED_BATCH:
 	case SCHED_IDLE:
-	case SCHED_EXT:
+#ifdef CONFIG_HMBIRD_SCHED
+	case SCHED_HMBIRD:
+#endif
 		ret = 0;
 	}
 	return ret;
@@ -9621,15 +9747,25 @@ int migrate_task_to(struct task_struct *p, int target_cpu)
  */
 void sched_setnuma(struct task_struct *p, int nid)
 {
+	bool queued, running;
 	struct rq_flags rf;
 	struct rq *rq;
 
 	rq = task_rq_lock(p, &rf);
+	queued = task_on_rq_queued(p);
+	running = task_current(rq, p);
 
-	SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE) {
-		p->numa_preferred_nid = nid;
-	}
+	if (queued)
+		dequeue_task(rq, p, DEQUEUE_SAVE);
+	if (running)
+		put_prev_task(rq, p);
+
+	p->numa_preferred_nid = nid;
 
+	if (queued)
+		enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
+	if (running)
+		set_next_task(rq, p);
 	task_rq_unlock(rq, p, &rf);
 }
 #endif /* CONFIG_NUMA_BALANCING */
@@ -10195,15 +10331,22 @@ void __init sched_init(void)
 	int i;
 
 	/* Make sure the linker didn't screw up */
+#ifdef CONFIG_HMBIRD_SCHED
+#ifdef CONFIG_SMP
+	WARN_ON_ONCE(!sched_class_above(&stop_sched_class, &dl_sched_class));
+#endif
+	WARN_ON_ONCE(!sched_class_above(&dl_sched_class, &rt_sched_class));
+	WARN_ON_ONCE(!sched_class_above(&rt_sched_class, &fair_sched_class));
+	WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &idle_sched_class));
+	WARN_ON_ONCE(!sched_class_above(&fair_sched_class, &hmbird_sched_class));
+	WARN_ON_ONCE(!sched_class_above(&hmbird_sched_class, &idle_sched_class));
+#else
+	BUG_ON(&idle_sched_class != &fair_sched_class + 1 ||
+	       &fair_sched_class != &rt_sched_class + 1 ||
+	       &rt_sched_class   != &dl_sched_class + 1);
 #ifdef CONFIG_SMP
-	BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class));
+	BUG_ON(&dl_sched_class != &stop_sched_class + 1);
 #endif
-	BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class));
-	BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class));
-	BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class));
-#ifdef CONFIG_SCHED_CLASS_EXT
-	BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class));
-	BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class));
 #endif
 
 	wait_bit_init();
@@ -10374,8 +10517,9 @@ void __init sched_init(void)
 	balance_push_set(smp_processor_id(), false);
 #endif
 	init_sched_fair_class();
-	init_sched_ext_class();
-
+#ifdef CONFIG_HMBIRD_SCHED
+	init_sched_hmbird_class();
+#endif
 	psi_init();
 
 	init_uclamp();
@@ -10785,6 +10929,8 @@ static void sched_change_group(struct task_struct *tsk, struct task_group *group
  */
 void sched_move_task(struct task_struct *tsk)
 {
+	int queued, running, queue_flags =
+		DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
 	struct task_group *group;
 	struct rq_flags rf;
 	struct rq *rq;
@@ -10801,18 +10947,27 @@ void sched_move_task(struct task_struct *tsk)
 
 	update_rq_clock(rq);
 
-	SCHED_CHANGE_BLOCK(rq, tsk,
-			   DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
-		sched_change_group(tsk, group);
-	}
+	running = task_current(rq, tsk);
+	queued = task_on_rq_queued(tsk);
 
-	/*
-	 * After changing group, the running task may have joined a throttled
-	 * one but it's still the running task. Trigger a resched to make sure
-	 * that task can still run.
-	 */
-	if (task_current(rq, tsk))
+	if (queued)
+		dequeue_task(rq, tsk, queue_flags);
+	if (running)
+		put_prev_task(rq, tsk);
+
+	sched_change_group(tsk, group);
+
+	if (queued)
+		enqueue_task(rq, tsk, queue_flags);
+	if (running) {
+		set_next_task(rq, tsk);
+		/*
+		 * After changing group, the running task may have joined a
+		 * throttled one but it's still the running task. Trigger a
+		 * resched to make sure that task can still run.
+		 */
 		resched_curr(rq);
+	}
 
 unlock:
 	task_rq_unlock(rq, tsk, &rf);
@@ -10850,6 +11005,10 @@ static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
 	struct task_group *tg = css_tg(css);
 	struct task_group *parent = css_tg(css->parent);
 
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_tg_online(tg);
+#endif
+
 	if (parent)
 		sched_online_group(tg, parent);
 
@@ -10899,13 +11058,77 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
 }
 #endif
 
+#ifdef CONFIG_HMBIRD_SCHED
+static inline void update_cgroup_ids_table(int ids, u8 hmbird_cgroup_deadline_idx)
+{
+	if (ids < 0 || ids >= NUMS_CGROUP_KINDS) {
+		pr_err("update_cgroup_ids_tab idx err!\n");
+		return;
+	}
+	cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx;
+}
+
+static int cgroup_write_hmbird_deadline(struct cgroup_subsys_state *css,
+						struct cftype *cftype, u64 dl)
+{
+	int i;
+
+	for (i = MIN_CGROUP_DL_IDX; i < MAX_GLOBAL_DSQS; ++i) {
+		if (dl < HMBIRD_BPF_DSQS_DEADLINE[i])
+			break;
+	}
+
+	i = max_t(int, i-1, MIN_CGROUP_DL_IDX);
+	if (!css || !css->cgroup || !css->cgroup->kn)
+		return 0;
+	update_cgroup_ids_table(css->cgroup->kn->id, i);
+
+	return 0;
+}
+
+static u64 cgroup_read_hmbird_deadline(struct cgroup_subsys_state *css,
+						struct cftype *cft)
+{
+	u8 i;
+
+	if (!css || !css->cgroup || !css->cgroup->kn)
+		return (u64) HMBIRD_BPF_DSQS_DEADLINE[DEFAULT_CGROUP_DL_IDX];
+	i = min_t(u8, cgroup_ids_table[css->cgroup->kn->id], MAX_GLOBAL_DSQS-1);
+	if (i < 0) {
+		pr_err("<sched_ext> <%s> i is %d, less than 0, name is %s\n",
+			__func__, i, css->cgroup->kn->name);
+		i = DEFAULT_CGROUP_DL_IDX;
+	}
+
+	return (u64) HMBIRD_BPF_DSQS_DEADLINE[i];
+}
+
+static void hmbird_change_rt_sched_prop(struct cgroup_subsys_state *css,
+						struct task_struct *p, int prio)
+{
+	if (!css || !rt_prio(prio))
+		return;
+
+	if (!(hmbird_get_sched_prop(p) & SCHED_PROP_DEADLINE_MASK)) {
+		if (!strcmp(css->cgroup->kn->name, "display"))
+			hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL3);
+		else if (!strcmp(css->cgroup->kn->name, "touch"))
+			hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL2);
+		else if (!strcmp(css->cgroup->kn->name, "multimedia"))
+			hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1);
+	}
+}
+#endif
+
 static void cpu_cgroup_attach(struct cgroup_taskset *tset)
 {
 	struct task_struct *task;
 	struct cgroup_subsys_state *css;
 
 	cgroup_taskset_for_each(task, css, tset) {
-
+#ifdef CONFIG_HMBIRD_SCHED
+		hmbird_change_rt_sched_prop(css, task, task->prio);
+#endif
 		sched_move_task(task);
 	}
 
@@ -11587,7 +11810,13 @@ static struct cftype cpu_legacy_files[] = {
 		.write_u64 = cpu_uclamp_ls_write_u64,
 	},
 #endif
-
+#ifdef CONFIG_HMBIRD_SCHED
+	{
+		.name = "hmbird.deadline",
+		.read_u64 = cgroup_read_hmbird_deadline,
+		.write_u64 = cgroup_write_hmbird_deadline,
+	},
+#endif
 	{ }	/* Terminate */
 };
 
@@ -11636,7 +11865,6 @@ static int cpu_local_stat_show(struct seq_file *sf,
 }
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-
 static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
 			       struct cftype *cft)
 {
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 423404100c9a..2fe1a734e640 100755
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -376,8 +376,8 @@ static __init int sched_init_debug(void)
 
 	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
 
-#ifdef CONFIG_SCHED_CLASS_EXT
-	debugfs_create_file("ext", 0444, debugfs_sched, NULL, &sched_ext_fops);
+#ifdef CONFIG_HMBIRD_SCHED
+	debugfs_create_file("hmbird", 0444, debugfs_sched, NULL, &sched_hmbird_fops);
 #endif
 	return 0;
 }
@@ -1006,9 +1006,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
 	SEQ_printf(m,
 		"---------------------------------------------------------"
 		"----------\n");
-#ifdef CONFIG_SLIM_SCHED
-	SEQ_printf(m, "p->sched_prop:0x%lx\n", p->sched_prop);
-#endif
 
 #define P_SCHEDSTAT(F)  __PS(#F, schedstat_val(p->stats.F))
 #define PN_SCHEDSTAT(F) __PSN(#F, schedstat_val(p->stats.F))
@@ -1102,8 +1099,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
 		P(dl.runtime);
 		P(dl.deadline);
 	}
-#ifdef CONFIG_SCHED_CLASS_EXT
-	__PS("ext.enabled", p->sched_class == &ext_sched_class);
+#ifdef CONFIG_HMBIRD_SCHED
+	__PS("hmbird.enabled", p->sched_class == &hmbird_sched_class);
+	__PS("sched_prop", get_hmbird_ts(p)->sched_prop);
+	__PS("top_task_prop", get_hmbird_ts(p)->top_task_prop);
 #endif
 #undef PN_SCHEDSTAT
 #undef P_SCHEDSTAT
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
deleted file mode 100755
index 4845d441df2b..000000000000
--- a/kernel/sched/ext.c
+++ /dev/null
@@ -1,3978 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#define SCX_OP_IDX(op)		(offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))
-
-enum scx_internal_consts {
-	SCX_NR_ONLINE_OPS	= SCX_OP_IDX(init),
-	SCX_DSP_DFL_MAX_BATCH	= 32,
-	SCX_DSP_MAX_LOOPS	= 32,
-	SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
-};
-
-enum scx_ops_enable_state {
-	SCX_OPS_PREPPING,
-	SCX_OPS_ENABLING,
-	SCX_OPS_ENABLED,
-	SCX_OPS_DISABLING,
-	SCX_OPS_DISABLED,
-};
-
-static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
-{
-	return css ? container_of(css, struct task_group, css) : NULL;
-}
-
-/*
- * sched_ext_entity->ops_state
- *
- * Used to track the task ownership between the SCX core and the BPF scheduler.
- * State transitions look as follows:
- *
- * NONE -> QUEUEING -> QUEUED -> DISPATCHING
- *   ^              |                 |
- *   |              v                 v
- *   \-------------------------------/
- *
- * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call
- * sites for explanations on the conditions being waited upon and why they are
- * safe. Transitions out of them into NONE or QUEUED must store_release and the
- * waiters should load_acquire.
- *
- * Tracking scx_ops_state enables sched_ext core to reliably determine whether
- * any given task can be dispatched by the BPF scheduler at all times and thus
- * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler
- * to try to dispatch any task anytime regardless of its state as the SCX core
- * can safely reject invalid dispatches.
- */
-enum scx_ops_state {
-	SCX_OPSS_NONE,		/* owned by the SCX core */
-	SCX_OPSS_QUEUEING,	/* in transit to the BPF scheduler */
-	SCX_OPSS_QUEUED,	/* owned by the BPF scheduler */
-	SCX_OPSS_DISPATCHING,	/* in transit back to the SCX core */
-
-	/*
-	 * QSEQ brands each QUEUED instance so that, when dispatch races
-	 * dequeue/requeue, the dispatcher can tell whether it still has a claim
-	 * on the task being dispatched.
-	 */
-	SCX_OPSS_QSEQ_SHIFT	= 2,
-	SCX_OPSS_STATE_MASK	= (1LLU << SCX_OPSS_QSEQ_SHIFT) - 1,
-	SCX_OPSS_QSEQ_MASK	= ~SCX_OPSS_STATE_MASK,
-};
-
-/*
- * During exit, a task may schedule after losing its PIDs. When disabling the
- * BPF scheduler, we need to be able to iterate tasks in every state to
- * guarantee system safety. Maintain a dedicated task list which contains every
- * task between its fork and eventual free.
- */
-static DEFINE_SPINLOCK(scx_tasks_lock);
-static LIST_HEAD(scx_tasks);
-
-/* ops enable/disable */
-static struct kthread_worker *scx_ops_helper;
-static DEFINE_MUTEX(scx_ops_enable_mutex);
-DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled);
-DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
-static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED);
-static bool scx_switch_all_req;
-static bool scx_switching_all;
-DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
-
-static struct sched_ext_ops scx_ops;
-static bool scx_warned_zero_slice;
-
-static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);
-static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);
-DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);
-static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
-
-struct static_key_false scx_has_op[SCX_NR_ONLINE_OPS] =
-	{ [0 ... SCX_NR_ONLINE_OPS-1] = STATIC_KEY_FALSE_INIT };
-
-static atomic_t scx_exit_type = ATOMIC_INIT(SCX_EXIT_DONE);
-static struct scx_exit_info scx_exit_info;
-
-static atomic64_t scx_nr_rejected = ATOMIC64_INIT(0);
-
-/*
- * The maximum amount of time in jiffies that a task may be runnable without
- * being scheduled on a CPU. If this timeout is exceeded, it will trigger
- * scx_ops_error().
- */
-unsigned long scx_watchdog_timeout;
-
-/*
- * The last time the delayed work was run. This delayed work relies on
- * ksoftirqd being able to run to service timer interrupts, so it's possible
- * that this work itself could get wedged. To account for this, we check that
- * it's not stalled in the timer tick, and trigger an error if it is.
- */
-unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES;
-
-static struct delayed_work scx_watchdog_work;
-
-/* idle tracking */
-#ifdef CONFIG_SMP
-#ifdef CONFIG_CPUMASK_OFFSTACK
-#define CL_ALIGNED_IF_ONSTACK
-#else
-#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp
-#endif
-
-static struct {
-	cpumask_var_t cpu;
-	cpumask_var_t smt;
-} idle_masks CL_ALIGNED_IF_ONSTACK;
-
-static bool __cacheline_aligned_in_smp scx_has_idle_cpus;
-#endif	/* CONFIG_SMP */
-
-/* for %SCX_KICK_WAIT */
-static u64 __percpu *scx_kick_cpus_pnt_seqs;
-
-/*
- * Direct dispatch marker.
- *
- * Non-NULL values are used for direct dispatch from enqueue path. A valid
- * pointer points to the task currently being enqueued. An ERR_PTR value is used
- * to indicate that direct dispatch has already happened.
- */
-static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
-
-/* dispatch queues */
-static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global;
-
-static LLIST_HEAD(dsqs_to_free);
-
-/* dispatch buf */
-struct scx_dsp_buf_ent {
-	struct task_struct	*task;
-	u64			qseq;
-	u64			dsq_id;
-	u64			enq_flags;
-};
-
-static u32 scx_dsp_max_batch;
-static struct scx_dsp_buf_ent __percpu *scx_dsp_buf;
-
-struct scx_dsp_ctx {
-	struct rq		*rq;
-	struct rq_flags		*rf;
-	u32			buf_cursor;
-	u32			nr_tasks;
-};
-
-static DEFINE_PER_CPU(struct scx_dsp_ctx, scx_dsp_ctx);
-
-void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice,
-		      u64 enq_flags);
-void scx_bpf_kick_cpu(s32 cpu, u64 flags);
-
-struct scx_task_iter {
-	struct sched_ext_entity		cursor;
-	struct task_struct		*locked;
-	struct rq			*rq;
-	struct rq_flags			rf;
-};
-
-#define SCX_HAS_OP(op)	static_branch_likely(&scx_has_op[SCX_OP_IDX(op)])
-
-/* if the highest set bit is N, return a mask with bits [N+1, 31] set */
-static u32 higher_bits(u32 flags)
-{
-	return ~((1 << fls(flags)) - 1);
-}
-
-/* return the mask with only the highest bit set */
-static u32 highest_bit(u32 flags)
-{
-	int bit = fls(flags);
-	return bit ? 1 << (bit - 1) : 0;
-}
-
-/*
- * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX
- * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate
- * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check
- * whether it's running from an allowed context.
- *
- * @mask is constant, always inline to cull the mask calculations.
- */
-static __always_inline void scx_kf_allow(u32 mask)
-{
-	/* nesting is allowed only in increasing scx_kf_mask order */
-	WARN_ONCE((mask | higher_bits(mask)) & current->scx->kf_mask,
-		  "invalid nesting current->scx->kf_mask=0x%x mask=0x%x\n",
-		  current->scx->kf_mask, mask);
-	current->scx->kf_mask |= mask;
-}
-
-static void scx_kf_disallow(u32 mask)
-{
-	current->scx->kf_mask &= ~mask;
-}
-
-#define SCX_CALL_OP(mask, op, args...)						\
-do {										\
-	if (mask) {								\
-		scx_kf_allow(mask);						\
-		scx_ops.op(args);						\
-		scx_kf_disallow(mask);						\
-	} else {								\
-		scx_ops.op(args);						\
-	}									\
-} while (0)
-
-#define SCX_CALL_OP_RET(mask, op, args...)					\
-({										\
-	__typeof__(scx_ops.op(args)) __ret;					\
-	if (mask) {								\
-		scx_kf_allow(mask);						\
-		__ret = scx_ops.op(args);					\
-		scx_kf_disallow(mask);						\
-	} else {								\
-		__ret = scx_ops.op(args);					\
-	}									\
-	__ret;									\
-})
-
-/*
- * Some kfuncs are allowed only on the tasks that are subjects of the
- * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such
- * restrictions, the following SCX_CALL_OP_*() variants should be used when
- * invoking scx_ops operations that take task arguments. These can only be used
- * for non-nesting operations due to the way the tasks are tracked.
- *
- * kfuncs which can only operate on such tasks can in turn use
- * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on
- * the specific task.
- */
-#define SCX_CALL_OP_TASK(mask, op, task, args...)				\
-do {										\
-	BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL);				\
-	current->scx->kf_tasks[0] = task;					\
-	SCX_CALL_OP(mask, op, task, ##args);					\
-	current->scx->kf_tasks[0] = NULL;					\
-} while (0)
-
-#define SCX_CALL_OP_TASK_RET(mask, op, task, args...)				\
-({										\
-	__typeof__(scx_ops.op(task, ##args)) __ret;				\
-	BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL);				\
-	current->scx->kf_tasks[0] = task;					\
-	__ret = SCX_CALL_OP_RET(mask, op, task, ##args);			\
-	current->scx->kf_tasks[0] = NULL;					\
-	__ret;									\
-})
-
-#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...)			\
-({										\
-	__typeof__(scx_ops.op(task0, task1, ##args)) __ret;			\
-	BUILD_BUG_ON(mask & ~__SCX_KF_TERMINAL);				\
-	current->scx->kf_tasks[0] = task0;					\
-	current->scx->kf_tasks[1] = task1;					\
-	__ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args);		\
-	current->scx->kf_tasks[0] = NULL;					\
-	current->scx->kf_tasks[1] = NULL;					\
-	__ret;									\
-})
-
-//#include "./slim_walt.c"
-
-/* @mask is constant, always inline to cull unnecessary branches */
-static __always_inline bool scx_kf_allowed(u32 mask)
-{
-	if (unlikely(!(current->scx->kf_mask & mask))) {
-		scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x",
-			      mask, current->scx->kf_mask);
-		return false;
-	}
-
-	if (unlikely((mask & (SCX_KF_INIT | SCX_KF_SLEEPABLE)) &&
-		     in_interrupt())) {
-		scx_ops_error("sleepable kfunc called from non-sleepable context");
-		return false;
-	}
-
-	/*
-	 * Enforce nesting boundaries. e.g. A kfunc which can be called from
-	 * DISPATCH must not be called if we're running DEQUEUE which is nested
-	 * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE
-	 * boundary thanks to the above in_interrupt() check.
-	 */
-	if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE &&
-		     (current->scx->kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) {
-		scx_ops_error("cpu_release kfunc called from a nested operation");
-		return false;
-	}
-
-	if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH &&
-		     (current->scx->kf_mask & higher_bits(SCX_KF_DISPATCH)))) {
-		scx_ops_error("dispatch kfunc called from a nested operation");
-		return false;
-	}
-
-	return true;
-}
-
-/* see SCX_CALL_OP_TASK() */
-static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask,
-							struct task_struct *p)
-{
-	if (!scx_kf_allowed(__SCX_KF_RQ_LOCKED))
-		return false;
-
-	if (unlikely((p != current->scx->kf_tasks[0] &&
-		      p != current->scx->kf_tasks[1]))) {
-		scx_ops_error("called on a task not being operated on");
-		return false;
-	}
-
-	return true;
-}
-
-/**
- * scx_task_iter_init - Initialize a task iterator
- * @iter: iterator to init
- *
- * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized,
- * @iter must eventually be exited with scx_task_iter_exit().
- *
- * scx_tasks_lock may be released between this and the first next() call or
- * between any two next() calls. If scx_tasks_lock is released between two
- * next() calls, the caller is responsible for ensuring that the task being
- * iterated remains accessible either through RCU read lock or obtaining a
- * reference count.
- *
- * All tasks which existed when the iteration started are guaranteed to be
- * visited as long as they still exist.
- */
-static void scx_task_iter_init(struct scx_task_iter *iter)
-{
-	lockdep_assert_held(&scx_tasks_lock);
-
-	iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
-	list_add(&iter->cursor.tasks_node, &scx_tasks);
-	iter->locked = NULL;
-}
-
-/**
- * scx_task_iter_exit - Exit a task iterator
- * @iter: iterator to exit
- *
- * Exit a previously initialized @iter. Must be called with scx_tasks_lock held.
- * If the iterator holds a task's rq lock, that rq lock is released. See
- * scx_task_iter_init() for details.
- */
-static void scx_task_iter_exit(struct scx_task_iter *iter)
-{
-	struct list_head *cursor = &iter->cursor.tasks_node;
-
-	lockdep_assert_held(&scx_tasks_lock);
-
-	if (iter->locked) {
-		task_rq_unlock(iter->rq, iter->locked, &iter->rf);
-		iter->locked = NULL;
-	}
-
-	if (list_empty(cursor))
-		return;
-
-	list_del_init(cursor);
-}
-
-/**
- * scx_task_iter_next - Next task
- * @iter: iterator to walk
- *
- * Visit the next task. See scx_task_iter_init() for details.
- */
-static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
-{
-	struct list_head *cursor = &iter->cursor.tasks_node;
-	struct sched_ext_entity *pos;
-
-	lockdep_assert_held(&scx_tasks_lock);
-
-	list_for_each_entry(pos, cursor, tasks_node) {
-		if (&pos->tasks_node == &scx_tasks)
-			return NULL;
-		if (!(pos->flags & SCX_TASK_CURSOR)) {
-			list_move(cursor, &pos->tasks_node);
-			return pos->task;
-		}
-	}
-
-	/* can't happen, should always terminate at scx_tasks above */
-	BUG();
-}
-
-/**
- * scx_task_iter_next_filtered - Next non-idle task
- * @iter: iterator to walk
- *
- * Visit the next non-idle task. See scx_task_iter_init() for details.
- */
-static struct task_struct *
-scx_task_iter_next_filtered(struct scx_task_iter *iter)
-{
-	struct task_struct *p;
-
-	while ((p = scx_task_iter_next(iter))) {
-		if (!is_idle_task(p))
-			return p;
-	}
-	return NULL;
-}
-
-/**
- * scx_task_iter_next_filtered_locked - Next non-idle task with its rq locked
- * @iter: iterator to walk
- *
- * Visit the next non-idle task with its rq lock held. See scx_task_iter_init()
- * for details.
- */
-static struct task_struct *
-scx_task_iter_next_filtered_locked(struct scx_task_iter *iter)
-{
-	struct task_struct *p;
-
-	if (iter->locked) {
-		task_rq_unlock(iter->rq, iter->locked, &iter->rf);
-		iter->locked = NULL;
-	}
-
-	p = scx_task_iter_next_filtered(iter);
-	if (!p)
-		return NULL;
-
-	iter->rq = task_rq_lock(p, &iter->rf);
-	iter->locked = p;
-	return p;
-}
-
-static enum scx_ops_enable_state scx_ops_enable_state(void)
-{
-	return atomic_read(&scx_ops_enable_state_var);
-}
-
-static enum scx_ops_enable_state
-scx_ops_set_enable_state(enum scx_ops_enable_state to)
-{
-	return atomic_xchg(&scx_ops_enable_state_var, to);
-}
-
-static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to,
-					enum scx_ops_enable_state from)
-{
-	int from_v = from;
-
-	return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to);
-}
-
-static bool scx_ops_disabling(void)
-{
-	return unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING);
-}
-
-/**
- * wait_ops_state - Busy-wait the specified ops state to end
- * @p: target task
- * @opss: state to wait the end of
- *
- * Busy-wait for @p to transition out of @opss. This can only be used when the
- * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also
- * has load_acquire semantics to ensure that the caller can see the updates made
- * in the enqueueing and dispatching paths.
- */
-static void wait_ops_state(struct task_struct *p, u64 opss)
-{
-	do {
-		cpu_relax();
-	} while (atomic64_read_acquire(&p->scx->ops_state) == opss);
-}
-
-/**
- * ops_cpu_valid - Verify a cpu number
- * @cpu: cpu number which came from a BPF ops
- *
- * @cpu is a cpu number which came from the BPF scheduler and can be any value.
- * Verify that it is in range and one of the possible cpus.
- */
-static bool ops_cpu_valid(s32 cpu)
-{
-	return likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu));
-}
-
-/**
- * ops_sanitize_err - Sanitize a -errno value
- * @ops_name: operation to blame on failure
- * @err: -errno value to sanitize
- *
- * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return
- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can
- * cause misbehaviors. For an example, a large negative return from
- * ops.prep_enable() triggers an oops when passed up the call chain because the
- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is
- * handled as a pointer.
- */
-static int ops_sanitize_err(const char *ops_name, s32 err)
-{
-	if (err < 0 && err >= -MAX_ERRNO)
-		return err;
-
-	scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err);
-	return -EPROTO;
-}
-
-/**
- * touch_core_sched - Update timestamp used for core-sched task ordering
- * @rq: rq to read clock from, must be locked
- * @p: task to update the timestamp for
- *
- * Update @p->scx->core_sched_at timestamp. This is used by scx_prio_less() to
- * implement global or local-DSQ FIFO ordering for core-sched. Should be called
- * when a task becomes runnable and its turn on the CPU ends (e.g. slice
- * exhaustion).
- */
-static void touch_core_sched(struct rq *rq, struct task_struct *p)
-{
-#ifdef CONFIG_SCHED_CORE
-	/*
-	 * It's okay to update the timestamp spuriously. Use
-	 * sched_core_disabled() which is cheaper than enabled().
-	 */
-	if (!sched_core_disabled())
-		p->scx->core_sched_at = rq_clock_task(rq);
-#endif
-}
-
-/**
- * touch_core_sched_dispatch - Update core-sched timestamp on dispatch
- * @rq: rq to read clock from, must be locked
- * @p: task being dispatched
- *
- * If the BPF scheduler implements custom core-sched ordering via
- * ops.core_sched_before(), @p->scx->core_sched_at is used to implement FIFO
- * ordering within each local DSQ. This function is called from dispatch paths
- * and updates @p->scx->core_sched_at if custom core-sched ordering is in effect.
- */
-static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p)
-{
-	lockdep_assert_rq_held(rq);
-	assert_clock_updated(rq);
-
-#ifdef CONFIG_SCHED_CORE
-	if (SCX_HAS_OP(core_sched_before))
-		touch_core_sched(rq, p);
-#endif
-}
-
-static void update_curr_scx(struct rq *rq)
-{
-	struct task_struct *curr = rq->curr;
-	u64 now = rq_clock_task(rq);
-	u64 delta_exec;
-
-	if (time_before_eq64(now, curr->se.exec_start))
-		return;
-
-	delta_exec = now - curr->se.exec_start;
-	curr->se.exec_start = now;
-	curr->se.sum_exec_runtime += delta_exec;
-	account_group_exec_runtime(curr, delta_exec);
-	cgroup_account_cputime(curr, delta_exec);
-
-	if (curr->scx->slice != SCX_SLICE_INF) {
-		curr->scx->slice -= min(curr->scx->slice, delta_exec);
-		if (!curr->scx->slice)
-			touch_core_sched(rq, curr);
-	}
-}
-
-static bool scx_dsq_priq_less(struct rb_node *node_a,
-			      const struct rb_node *node_b)
-{
-	const struct sched_ext_entity *a =
-		container_of(node_a, struct sched_ext_entity, dsq_node.priq);
-	const struct sched_ext_entity *b =
-		container_of(node_b, struct sched_ext_entity, dsq_node.priq);
-
-	return time_before64(a->dsq_vtime, b->dsq_vtime);
-}
-
-static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p,
-			     u64 enq_flags)
-{
-	bool is_local = dsq->id == SCX_DSQ_LOCAL;
-
-	WARN_ON_ONCE(p->scx->dsq || !list_empty(&p->scx->dsq_node.fifo));
-	WARN_ON_ONCE((p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
-		     !RB_EMPTY_NODE(&p->scx->dsq_node.priq));
-
-	if (!is_local) {
-		raw_spin_lock(&dsq->lock);
-		if (unlikely(dsq->id == SCX_DSQ_INVALID)) {
-			scx_ops_error("attempting to dispatch to a destroyed dsq");
-			/* fall back to the global dsq */
-			raw_spin_unlock(&dsq->lock);
-			dsq = &scx_dsq_global;
-			raw_spin_lock(&dsq->lock);
-		}
-	}
-
-	if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
-		p->scx->dsq_flags |= SCX_TASK_DSQ_ON_PRIQ;
-		rb_add_cached(&p->scx->dsq_node.priq, &dsq->priq,
-			      scx_dsq_priq_less);
-	} else {
-		if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
-			list_add(&p->scx->dsq_node.fifo, &dsq->fifo);
-		else
-			list_add_tail(&p->scx->dsq_node.fifo, &dsq->fifo);
-	}
-	dsq->nr++;
-	p->scx->dsq = dsq;
-
-	/*
-	 * We're transitioning out of QUEUEING or DISPATCHING. store_release to
-	 * match waiters' load_acquire.
-	 */
-	if (enq_flags & SCX_ENQ_CLEAR_OPSS)
-		atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE);
-
-	if (is_local) {
-		struct scx_rq *scx = container_of(dsq, struct scx_rq, local_dsq);
-		struct rq *rq = scx->rq;
-		bool preempt = false;
-
-		if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
-		    rq->curr->sched_class == &ext_sched_class) {
-			rq->curr->scx->slice = 0;
-			preempt = true;
-		}
-
-		if (preempt || sched_class_above(&ext_sched_class,
-						 rq->curr->sched_class))
-			resched_curr(rq);
-	} else {
-		raw_spin_unlock(&dsq->lock);
-	}
-}
-
-static void task_unlink_from_dsq(struct task_struct *p,
-				 struct scx_dispatch_q *dsq)
-{
-	if (p->scx->dsq_flags & SCX_TASK_DSQ_ON_PRIQ) {
-		rb_erase_cached(&p->scx->dsq_node.priq, &dsq->priq);
-		RB_CLEAR_NODE(&p->scx->dsq_node.priq);
-		p->scx->dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ;
-	} else {
-		list_del_init(&p->scx->dsq_node.fifo);
-	}
-
-}
-
-static bool task_linked_on_dsq(struct task_struct *p)
-{
-	return !list_empty(&p->scx->dsq_node.fifo) ||
-		!RB_EMPTY_NODE(&p->scx->dsq_node.priq);
-}
-
-static void dispatch_dequeue(struct scx_rq *scx_rq, struct task_struct *p)
-{
-	struct scx_dispatch_q *dsq = p->scx->dsq;
-	bool is_local = dsq == &scx_rq->local_dsq;
-
-	if (!dsq) {
-		WARN_ON_ONCE(task_linked_on_dsq(p));
-		/*
-		 * When dispatching directly from the BPF scheduler to a local
-		 * DSQ, the task isn't associated with any DSQ but
-		 * @p->scx->holding_cpu may be set under the protection of
-		 * %SCX_OPSS_DISPATCHING.
-		 */
-		if (p->scx->holding_cpu >= 0)
-			p->scx->holding_cpu = -1;
-		return;
-	}
-
-	if (!is_local)
-		raw_spin_lock(&dsq->lock);
-
-	/*
-	 * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx->dsq_node
-	 * can't change underneath us.
-	*/
-	if (p->scx->holding_cpu < 0) {
-		/* @p must still be on @dsq, dequeue */
-		WARN_ON_ONCE(!task_linked_on_dsq(p));
-		task_unlink_from_dsq(p, dsq);
-		dsq->nr--;
-	} else {
-		/*
-		 * We're racing against dispatch_to_local_dsq() which already
-		 * removed @p from @dsq and set @p->scx->holding_cpu. Clear the
-		 * holding_cpu which tells dispatch_to_local_dsq() that it lost
-		 * the race.
-		 */
-		WARN_ON_ONCE(task_linked_on_dsq(p));
-		p->scx->holding_cpu = -1;
-	}
-	p->scx->dsq = NULL;
-
-	if (!is_local)
-		raw_spin_unlock(&dsq->lock);
-}
-
-static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id)
-{
-	lockdep_assert(rcu_read_lock_any_held());
-
-	return &scx_dsq_global;
-}
-
-static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id,
-						    struct task_struct *p)
-{
-	struct scx_dispatch_q *dsq;
-
-	if (dsq_id == SCX_DSQ_LOCAL)
-		return &rq->scx->local_dsq;
-
-	dsq = find_non_local_dsq(dsq_id);
-	if (unlikely(!dsq)) {
-		scx_ops_error("non-existent DSQ 0x%llx for %s[%d]",
-			      dsq_id, p->comm, p->pid);
-		return &scx_dsq_global;
-	}
-
-	return dsq;
-}
-
-static void direct_dispatch(struct task_struct *ddsp_task, struct task_struct *p,
-			    u64 dsq_id, u64 enq_flags)
-{
-	struct scx_dispatch_q *dsq;
-
-	/* @p must match the task which is being enqueued */
-	if (unlikely(p != ddsp_task)) {
-		if (IS_ERR(ddsp_task))
-			scx_ops_error("%s[%d] already direct-dispatched",
-				      p->comm, p->pid);
-		else
-			scx_ops_error("enqueueing %s[%d] but trying to direct-dispatch %s[%d]",
-				      ddsp_task->comm, ddsp_task->pid,
-				      p->comm, p->pid);
-		return;
-	}
-
-	/*
-	 * %SCX_DSQ_LOCAL_ON is not supported during direct dispatch because
-	 * dispatching to the local DSQ of a different CPU requires unlocking
-	 * the current rq which isn't allowed in the enqueue path. Use
-	 * ops.select_cpu() to be on the target CPU and then %SCX_DSQ_LOCAL.
-	 */
-	if (unlikely((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON)) {
-		scx_ops_error("SCX_DSQ_LOCAL_ON can't be used for direct-dispatch");
-		return;
-	}
-
-	touch_core_sched_dispatch(task_rq(p), p);
-
-	dsq = find_dsq_for_dispatch(task_rq(p), dsq_id, p);
-	dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
-
-	/*
-	 * Mark that dispatch already happened by spoiling direct_dispatch_task
-	 * with a non-NULL value which can never match a valid task pointer.
-	 */
-	__this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
-}
-
-static bool test_rq_online(struct rq *rq)
-{
-#ifdef CONFIG_SMP
-	return rq->online;
-#else
-	return true;
-#endif
-}
-
-static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
-			    int sticky_cpu)
-{
-	struct task_struct **ddsp_taskp;
-	u64 qseq;
-
-	WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED));
-
-	if (p->scx->flags & SCX_TASK_ENQ_LOCAL) {
-		enq_flags |= SCX_ENQ_LOCAL;
-		p->scx->flags &= ~SCX_TASK_ENQ_LOCAL;
-	}
-
-	/* rq migration */
-	if (sticky_cpu == cpu_of(rq))
-		goto local_norefill;
-
-	/*
-	 * If !rq->online, we already told the BPF scheduler that the CPU is
-	 * offline. We're just trying to on/offline the CPU. Don't bother the
-	 * BPF scheduler.
-	 */
-	if (unlikely(!test_rq_online(rq)))
-		goto local;
-
-	/* see %SCX_OPS_ENQ_EXITING */
-	if (!static_branch_unlikely(&scx_ops_enq_exiting) &&
-	    unlikely(p->flags & PF_EXITING))
-		goto local;
-
-	/* see %SCX_OPS_ENQ_LAST */
-	if (!static_branch_unlikely(&scx_ops_enq_last) &&
-	    (enq_flags & SCX_ENQ_LAST))
-		goto local;
-
-	if (!SCX_HAS_OP(enqueue)) {
-		if (enq_flags & SCX_ENQ_LOCAL)
-			goto local;
-		else
-			goto global;
-	}
-
-	/* DSQ bypass didn't trigger, enqueue on the BPF scheduler */
-	qseq = rq->scx->ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
-
-	WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE);
-	atomic64_set(&p->scx->ops_state, SCX_OPSS_QUEUEING | qseq);
-
-	ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
-	WARN_ON_ONCE(*ddsp_taskp);
-	*ddsp_taskp = p;
-
-	SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags);
-
-	/*
-	 * If not directly dispatched, QUEUEING isn't clear yet and dispatch or
-	 * dequeue may be waiting. The store_release matches their load_acquire.
-	 */
-	if (*ddsp_taskp == p)
-		atomic64_set_release(&p->scx->ops_state, SCX_OPSS_QUEUED | qseq);
-	*ddsp_taskp = NULL;
-	return;
-
-local:
-	/*
-	 * For task-ordering, slice refill must be treated as implying the end
-	 * of the current slice. Otherwise, the longer @p stays on the CPU, the
-	 * higher priority it becomes from scx_prio_less()'s POV.
-	 */
-	touch_core_sched(rq, p);
-	p->scx->slice = SCX_SLICE_DFL;
-local_norefill:
-	dispatch_enqueue(&rq->scx->local_dsq, p, enq_flags);
-	return;
-
-global:
-	touch_core_sched(rq, p);	/* see the comment in local: */
-	p->scx->slice = SCX_SLICE_DFL;
-	dispatch_enqueue(&scx_dsq_global, p, enq_flags);
-}
-
-static bool watchdog_task_watched(const struct task_struct *p)
-{
-	return !list_empty(&p->scx->watchdog_node);
-}
-
-static void watchdog_watch_task(struct rq *rq, struct task_struct *p)
-{
-	lockdep_assert_rq_held(rq);
-	if (p->scx->flags & SCX_TASK_WATCHDOG_RESET)
-		p->scx->runnable_at = jiffies;
-	p->scx->flags &= ~SCX_TASK_WATCHDOG_RESET;
-	list_add_tail(&p->scx->watchdog_node, &rq->scx->watchdog_list);
-}
-
-static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout)
-{
-	list_del_init(&p->scx->watchdog_node);
-	if (reset_timeout)
-		p->scx->flags |= SCX_TASK_WATCHDOG_RESET;
-}
-
-static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags)
-{
-	int sticky_cpu = p->scx->sticky_cpu;
-
-	enq_flags |= rq->scx->extra_enq_flags;
-
-	if (sticky_cpu >= 0)
-		p->scx->sticky_cpu = -1;
-
-	/*
-	 * Restoring a running task will be immediately followed by
-	 * set_next_task_scx() which expects the task to not be on the BPF
-	 * scheduler as tasks can only start running through local DSQs. Force
-	 * direct-dispatch into the local DSQ by setting the sticky_cpu.
-	 */
-	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p))
-		sticky_cpu = cpu_of(rq);
-
-	if (p->scx->flags & SCX_TASK_QUEUED) {
-		WARN_ON_ONCE(!watchdog_task_watched(p));
-		return;
-	}
-
-	watchdog_watch_task(rq, p);
-	p->scx->flags |= SCX_TASK_QUEUED;
-	rq->scx->nr_running++;
-	add_nr_running(rq, 1);
-
-	if (SCX_HAS_OP(runnable))
-		SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags);
-
-	if (enq_flags & SCX_ENQ_WAKEUP)
-		touch_core_sched(rq, p);
-
-	do_enqueue_task(rq, p, enq_flags, sticky_cpu);
-}
-
-static void ops_dequeue(struct task_struct *p, u64 deq_flags)
-{
-	u64 opss;
-
-	watchdog_unwatch_task(p, false);
-
-	/* acquire ensures that we see the preceding updates on QUEUED */
-	opss = atomic64_read_acquire(&p->scx->ops_state);
-
-	switch (opss & SCX_OPSS_STATE_MASK) {
-	case SCX_OPSS_NONE:
-		break;
-	case SCX_OPSS_QUEUEING:
-		/*
-		 * QUEUEING is started and finished while holding @p's rq lock.
-		 * As we're holding the rq lock now, we shouldn't see QUEUEING.
-		 */
-		BUG();
-	case SCX_OPSS_QUEUED:
-		if (SCX_HAS_OP(dequeue))
-			SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags);
-
-		if (atomic64_try_cmpxchg(&p->scx->ops_state, &opss,
-					 SCX_OPSS_NONE))
-			break;
-		fallthrough;
-	case SCX_OPSS_DISPATCHING:
-		/*
-		 * If @p is being dispatched from the BPF scheduler to a DSQ,
-		 * wait for the transfer to complete so that @p doesn't get
-		 * added to its DSQ after dequeueing is complete.
-		 *
-		 * As we're waiting on DISPATCHING with the rq locked, the
-		 * dispatching side shouldn't try to lock the rq while
-		 * DISPATCHING is set. See dispatch_to_local_dsq().
-		 *
-		 * DISPATCHING shouldn't have qseq set and control can reach
-		 * here with NONE @opss from the above QUEUED case block.
-		 * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss.
-		 */
-		wait_ops_state(p, SCX_OPSS_DISPATCHING);
-		BUG_ON(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE);
-		break;
-	}
-}
-
-static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags)
-{
-	struct scx_rq *scx_rq = rq->scx;
-
-	if (!(p->scx->flags & SCX_TASK_QUEUED)) {
-		WARN_ON_ONCE(watchdog_task_watched(p));
-		return;
-	}
-
-	ops_dequeue(p, deq_flags);
-
-	/*
-	 * A currently running task which is going off @rq first gets dequeued
-	 * and then stops running. As we want running <-> stopping transitions
-	 * to be contained within runnable <-> quiescent transitions, trigger
-	 * ->stopping() early here instead of in put_prev_task_scx().
-	 *
-	 * @p may go through multiple stopping <-> running transitions between
-	 * here and put_prev_task_scx() if task attribute changes occur while
-	 * balance_scx() leaves @rq unlocked. However, they don't contain any
-	 * information meaningful to the BPF scheduler and can be suppressed by
-	 * skipping the callbacks if the task is !QUEUED.
-	 */
-	if (SCX_HAS_OP(stopping) && task_current(rq, p)) {
-		update_curr_scx(rq);
-		SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false);
-	}
-
-	//if(task_current(rq, p))
-	//	scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock);
-
-	if (SCX_HAS_OP(quiescent))
-		SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags);
-
-	if (deq_flags & SCX_DEQ_SLEEP)
-		p->scx->flags |= SCX_TASK_DEQD_FOR_SLEEP;
-	else
-		p->scx->flags &= ~SCX_TASK_DEQD_FOR_SLEEP;
-
-	p->scx->flags &= ~SCX_TASK_QUEUED;
-	BUG_ON(!scx_rq->nr_running);
-	scx_rq->nr_running--;
-	sub_nr_running(rq, 1);
-
-	dispatch_dequeue(scx_rq, p);
-}
-
-static void yield_task_scx(struct rq *rq)
-{
-	struct task_struct *p = rq->curr;
-
-	if (SCX_HAS_OP(yield))
-		SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL);
-	else
-		p->scx->slice = 0;
-}
-
-static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
-{
-	struct task_struct *from = rq->curr;
-
-	if (SCX_HAS_OP(yield))
-		return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to);
-	else
-		return false;
-}
-
-#ifdef CONFIG_SMP
-/**
- * move_task_to_local_dsq - Move a task from a different rq to a local DSQ
- * @rq: rq to move the task into, currently locked
- * @p: task to move
- * @enq_flags: %SCX_ENQ_*
- *
- * Move @p which is currently on a different rq to @rq's local DSQ. The caller
- * must:
- *
- * 1. Start with exclusive access to @p either through its DSQ lock or
- *    %SCX_OPSS_DISPATCHING flag.
- *
- * 2. Set @p->scx->holding_cpu to raw_smp_processor_id().
- *
- * 3. Remember task_rq(@p). Release the exclusive access so that we don't
- *    deadlock with dequeue.
- *
- * 4. Lock @rq and the task_rq from #3.
- *
- * 5. Call this function.
- *
- * Returns %true if @p was successfully moved. %false after racing dequeue and
- * losing.
- */
-static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p,
-				   u64 enq_flags)
-{
-	struct rq *task_rq;
-
-	lockdep_assert_rq_held(rq);
-
-	/*
-	 * If dequeue got to @p while we were trying to lock both rq's, it'd
-	 * have cleared @p->scx->holding_cpu to -1. While other cpus may have
-	 * updated it to different values afterwards, as this operation can't be
-	 * preempted or recurse, @p->scx->holding_cpu can never become
-	 * raw_smp_processor_id() again before we're done. Thus, we can tell
-	 * whether we lost to dequeue by testing whether @p->scx->holding_cpu is
-	 * still raw_smp_processor_id().
-	 *
-	 * See dispatch_dequeue() for the counterpart.
-	 */
-	if (unlikely(p->scx->holding_cpu != raw_smp_processor_id()))
-		return false;
-
-	/* @p->rq couldn't have changed if we're still the holding cpu */
-	task_rq = task_rq(p);
-	lockdep_assert_rq_held(task_rq);
-
-	WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr));
-	deactivate_task(task_rq, p, 0);
-	set_task_cpu(p, cpu_of(rq));
-	p->scx->sticky_cpu = cpu_of(rq);
-
-	/*
-	 * We want to pass scx-specific enq_flags but activate_task() will
-	 * truncate the upper 32 bit. As we own @rq, we can pass them through
-	 * @rq->scx->extra_enq_flags instead.
-	 */
-	WARN_ON_ONCE(rq->scx->extra_enq_flags);
-	rq->scx->extra_enq_flags = enq_flags;
-	activate_task(rq, p, 0);
-	rq->scx->extra_enq_flags = 0;
-
-	return true;
-}
-
-/**
- * dispatch_to_local_dsq_lock - Ensure source and desitnation rq's are locked
- * @rq: current rq which is locked
- * @rf: rq_flags to use when unlocking @rq
- * @src_rq: rq to move task from
- * @dst_rq: rq to move task to
- *
- * We're holding @rq lock and trying to dispatch a task from @src_rq to
- * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether
- * @rq stays locked isn't important as long as the state is restored after
- * dispatch_to_local_dsq_unlock().
- */
-static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq_flags *rf,
-				       struct rq *src_rq, struct rq *dst_rq)
-{
-	rq_unpin_lock(rq, rf);
-
-	if (src_rq == dst_rq) {
-		raw_spin_rq_unlock(rq);
-		raw_spin_rq_lock(dst_rq);
-	} else if (rq == src_rq) {
-		double_lock_balance(rq, dst_rq);
-		rq_repin_lock(rq, rf);
-	} else if (rq == dst_rq) {
-		double_lock_balance(rq, src_rq);
-		rq_repin_lock(rq, rf);
-	} else {
-		raw_spin_rq_unlock(rq);
-		double_rq_lock(src_rq, dst_rq);
-	}
-}
-
-/**
- * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock()
- * @rq: current rq which is locked
- * @rf: rq_flags to use when unlocking @rq
- * @src_rq: rq to move task from
- * @dst_rq: rq to move task to
- *
- * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return.
- */
-static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq_flags *rf,
-					 struct rq *src_rq, struct rq *dst_rq)
-{
-	if (src_rq == dst_rq) {
-		raw_spin_rq_unlock(dst_rq);
-		raw_spin_rq_lock(rq);
-		rq_repin_lock(rq, rf);
-	} else if (rq == src_rq) {
-		double_unlock_balance(rq, dst_rq);
-	} else if (rq == dst_rq) {
-		double_unlock_balance(rq, src_rq);
-	} else {
-		double_rq_unlock(src_rq, dst_rq);
-		raw_spin_rq_lock(rq);
-		rq_repin_lock(rq, rf);
-	}
-}
-#endif	/* CONFIG_SMP */
-
-
-static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq)
-{
-	return likely(test_rq_online(rq)) && !is_migration_disabled(p) &&
-		cpumask_test_cpu(cpu_of(rq), p->cpus_ptr);
-}
-
-static bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf,
-			       struct scx_dispatch_q *dsq)
-{
-	struct scx_rq *scx_rq = rq->scx;
-	struct sched_ext_entity *entity;
-	struct task_struct *p;
-	struct rb_node *rb_node;
-	struct rq *task_rq;
-	bool moved = false;
-retry:
-	if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq))
-		return false;
-
-	raw_spin_lock(&dsq->lock);
-
-	list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) {
-		p = entity->task;
-		task_rq = task_rq(p);
-		if (rq == task_rq)
-			goto this_rq;
-		if (task_can_run_on_rq(p, rq))
-			goto remote_rq;
-	}
-
-	for (rb_node = rb_first_cached(&dsq->priq); rb_node;
-	     rb_node = rb_next(rb_node)) {
-		entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq);
-		p = entity->task;
-		task_rq = task_rq(p);
-		if (rq == task_rq)
-			goto this_rq;
-		if (task_can_run_on_rq(p, rq))
-			goto remote_rq;
-	}
-
-	raw_spin_unlock(&dsq->lock);
-	return false;
-
-this_rq:
-	/* @dsq is locked and @p is on this rq */
-	WARN_ON_ONCE(p->scx->holding_cpu >= 0);
-	task_unlink_from_dsq(p, dsq);
-	list_add_tail(&p->scx->dsq_node.fifo, &scx_rq->local_dsq.fifo);
-	dsq->nr--;
-	scx_rq->local_dsq.nr++;
-	p->scx->dsq = &scx_rq->local_dsq;
-	raw_spin_unlock(&dsq->lock);
-	return true;
-
-remote_rq:
-#ifdef CONFIG_SMP
-	/*
-	 * @dsq is locked and @p is on a remote rq. @p is currently protected by
-	 * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab
-	 * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the
-	 * rq lock or fail, do a little dancing from our side. See
-	 * move_task_to_local_dsq().
-	 */
-	WARN_ON_ONCE(p->scx->holding_cpu >= 0);
-	task_unlink_from_dsq(p, dsq);
-	dsq->nr--;
-	p->scx->holding_cpu = raw_smp_processor_id();
-	raw_spin_unlock(&dsq->lock);
-
-	rq_unpin_lock(rq, rf);
-	double_lock_balance(rq, task_rq);
-	rq_repin_lock(rq, rf);
-
-	moved = move_task_to_local_dsq(rq, p, 0);
-
-	double_unlock_balance(rq, task_rq);
-#endif /* CONFIG_SMP */
-	if (likely(moved))
-		return true;
-	goto retry;
-}
-
-enum dispatch_to_local_dsq_ret {
-	DTL_DISPATCHED,		/* successfully dispatched */
-	DTL_LOST,		/* lost race to dequeue */
-	DTL_NOT_LOCAL,		/* destination is not a local DSQ */
-	DTL_INVALID,		/* invalid local dsq_id */
-};
-
-/**
- * dispatch_to_local_dsq - Dispatch a task to a local dsq
- * @rq: current rq which is locked
- * @rf: rq_flags to use when unlocking @rq
- * @dsq_id: destination dsq ID
- * @p: task to dispatch
- * @enq_flags: %SCX_ENQ_*
- *
- * We're holding @rq lock and want to dispatch @p to the local DSQ identified by
- * @dsq_id. This function performs all the synchronization dancing needed
- * because local DSQs are protected with rq locks.
- *
- * The caller must have exclusive ownership of @p (e.g. through
- * %SCX_OPSS_DISPATCHING).
- */
-static enum dispatch_to_local_dsq_ret
-dispatch_to_local_dsq(struct rq *rq, struct rq_flags *rf, u64 dsq_id,
-		      struct task_struct *p, u64 enq_flags)
-{
-	struct rq *src_rq = task_rq(p);
-	struct rq *dst_rq;
-
-	/*
-	 * We're synchronized against dequeue through DISPATCHING. As @p can't
-	 * be dequeued, its task_rq and cpus_allowed are stable too.
-	 */
-	if (dsq_id == SCX_DSQ_LOCAL) {
-		dst_rq = rq;
-	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
-		s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
-
-		if (!ops_cpu_valid(cpu)) {
-			scx_ops_error("invalid cpu %d in SCX_DSQ_LOCAL_ON verdict for %s[%d]",
-				      cpu, p->comm, p->pid);
-			return DTL_INVALID;
-		}
-		dst_rq = cpu_rq(cpu);
-	} else {
-		return DTL_NOT_LOCAL;
-	}
-
-	/* if dispatching to @rq that @p is already on, no lock dancing needed */
-	if (rq == src_rq && rq == dst_rq) {
-		dispatch_enqueue(&dst_rq->scx->local_dsq, p,
-				 enq_flags | SCX_ENQ_CLEAR_OPSS);
-		return DTL_DISPATCHED;
-	}
-
-#ifdef CONFIG_SMP
-	if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) {
-		struct rq *locked_dst_rq = dst_rq;
-		bool dsp;
-
-		/*
-		 * @p is on a possibly remote @src_rq which we need to lock to
-		 * move the task. If dequeue is in progress, it'd be locking
-		 * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq
-		 * lock while holding DISPATCHING.
-		 *
-		 * As DISPATCHING guarantees that @p is wholly ours, we can
-		 * pretend that we're moving from a DSQ and use the same
-		 * mechanism - mark the task under transfer with holding_cpu,
-		 * release DISPATCHING and then follow the same protocol.
-		 */
-		p->scx->holding_cpu = raw_smp_processor_id();
-
-		/* store_release ensures that dequeue sees the above */
-		atomic64_set_release(&p->scx->ops_state, SCX_OPSS_NONE);
-
-		dispatch_to_local_dsq_lock(rq, rf, src_rq, locked_dst_rq);
-
-		/*
-		 * We don't require the BPF scheduler to avoid dispatching to
-		 * offline CPUs mostly for convenience but also because CPUs can
-		 * go offline between scx_bpf_dispatch() calls and here. If @p
-		 * is destined to an offline CPU, queue it on its current CPU
-		 * instead, which should always be safe. As this is an allowed
-		 * behavior, don't trigger an ops error.
-		 */
-		if (unlikely(!test_rq_online(dst_rq)))
-			dst_rq = src_rq;
-
-		if (src_rq == dst_rq) {
-			/*
-			 * As @p is staying on the same rq, there's no need to
-			 * go through the full deactivate/activate cycle.
-			 * Optimize by abbreviating the operations in
-			 * move_task_to_local_dsq().
-			 */
-			dsp = p->scx->holding_cpu == raw_smp_processor_id();
-			if (likely(dsp)) {
-				p->scx->holding_cpu = -1;
-				dispatch_enqueue(&dst_rq->scx->local_dsq, p,
-						 enq_flags);
-			}
-		} else {
-			dsp = move_task_to_local_dsq(dst_rq, p, enq_flags);
-		}
-
-		/* if the destination CPU is idle, wake it up */
-		if (dsp && p->sched_class > dst_rq->curr->sched_class)
-			resched_curr(dst_rq);
-
-		dispatch_to_local_dsq_unlock(rq, rf, src_rq, locked_dst_rq);
-
-		return dsp ? DTL_DISPATCHED : DTL_LOST;
-	}
-#endif /* CONFIG_SMP */
-
-	scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]",
-		      cpu_of(dst_rq), p->comm, p->pid);
-	return DTL_INVALID;
-}
-
-/**
- * finish_dispatch - Asynchronously finish dispatching a task
- * @rq: current rq which is locked
- * @rf: rq_flags to use when unlocking @rq
- * @p: task to finish dispatching
- * @qseq_at_dispatch: qseq when @p started getting dispatched
- * @dsq_id: destination DSQ ID
- * @enq_flags: %SCX_ENQ_*
- *
- * Dispatching to local DSQs may need to wait for queueing to complete or
- * require rq lock dancing. As we don't wanna do either while inside
- * ops.dispatch() to avoid locking order inversion, we split dispatching into
- * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the
- * task and its qseq. Once ops.dispatch() returns, this function is called to
- * finish up.
- *
- * There is no guarantee that @p is still valid for dispatching or even that it
- * was valid in the first place. Make sure that the task is still owned by the
- * BPF scheduler and claim the ownership before dispatching.
- */
-static void finish_dispatch(struct rq *rq, struct rq_flags *rf,
-			    struct task_struct *p, u64 qseq_at_dispatch,
-			    u64 dsq_id, u64 enq_flags)
-{
-	struct scx_dispatch_q *dsq;
-	u64 opss;
-
-	touch_core_sched_dispatch(rq, p);
-retry:
-	/*
-	 * No need for _acquire here. @p is accessed only after a successful
-	 * try_cmpxchg to DISPATCHING.
-	 */
-	opss = atomic64_read(&p->scx->ops_state);
-
-	switch (opss & SCX_OPSS_STATE_MASK) {
-	case SCX_OPSS_DISPATCHING:
-	case SCX_OPSS_NONE:
-		/* someone else already got to it */
-		return;
-	case SCX_OPSS_QUEUED:
-		/*
-		 * If qseq doesn't match, @p has gone through at least one
-		 * dispatch/dequeue and re-enqueue cycle between
-		 * scx_bpf_dispatch() and here and we have no claim on it.
-		 */
-		if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch)
-			return;
-
-		/*
-		 * While we know @p is accessible, we don't yet have a claim on
-		 * it - the BPF scheduler is allowed to dispatch tasks
-		 * spuriously and there can be a racing dequeue attempt. Let's
-		 * claim @p by atomically transitioning it from QUEUED to
-		 * DISPATCHING.
-		 */
-		if (likely(atomic64_try_cmpxchg(&p->scx->ops_state, &opss,
-						SCX_OPSS_DISPATCHING)))
-			break;
-		goto retry;
-	case SCX_OPSS_QUEUEING:
-		/*
-		 * do_enqueue_task() is in the process of transferring the task
-		 * to the BPF scheduler while holding @p's rq lock. As we aren't
-		 * holding any kernel or BPF resource that the enqueue path may
-		 * depend upon, it's safe to wait.
-		 */
-		wait_ops_state(p, opss);
-		goto retry;
-	}
-
-	BUG_ON(!(p->scx->flags & SCX_TASK_QUEUED));
-
-	switch (dispatch_to_local_dsq(rq, rf, dsq_id, p, enq_flags)) {
-	case DTL_DISPATCHED:
-		break;
-	case DTL_LOST:
-		break;
-	case DTL_INVALID:
-		dsq_id = SCX_DSQ_GLOBAL;
-		fallthrough;
-	case DTL_NOT_LOCAL:
-		dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()),
-					    dsq_id, p);
-		dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
-		break;
-	}
-}
-
-static void flush_dispatch_buf(struct rq *rq, struct rq_flags *rf)
-{
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx);
-	u32 u;
-
-	for (u = 0; u < dspc->buf_cursor; u++) {
-		struct scx_dsp_buf_ent *ent = &this_cpu_ptr(scx_dsp_buf)[u];
-
-		finish_dispatch(rq, rf, ent->task, ent->qseq, ent->dsq_id,
-				ent->enq_flags);
-	}
-
-	dspc->nr_tasks += dspc->buf_cursor;
-	dspc->buf_cursor = 0;
-}
-
-static int balance_one(struct rq *rq, struct task_struct *prev,
-		       struct rq_flags *rf, bool local)
-{
-	struct scx_rq *scx_rq = rq->scx;
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx);
-	bool prev_on_scx = prev->sched_class == &ext_sched_class;
-	int nr_loops = SCX_DSP_MAX_LOOPS;
-
-	lockdep_assert_rq_held(rq);
-
-	if (static_branch_unlikely(&scx_ops_cpu_preempt) &&
-	    unlikely(rq->scx->cpu_released)) {
-		/*
-		 * If the previous sched_class for the current CPU was not SCX,
-		 * notify the BPF scheduler that it again has control of the
-		 * core. This callback complements ->cpu_release(), which is
-		 * emitted in scx_notify_pick_next_task().
-		 */
-		if (SCX_HAS_OP(cpu_acquire))
-			SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_acquire, cpu_of(rq),
-				    NULL);
-		rq->scx->cpu_released = false;
-	}
-
-	if (prev_on_scx) {
-		WARN_ON_ONCE(local && (prev->scx->flags & SCX_TASK_BAL_KEEP));
-		update_curr_scx(rq);
-
-		/*
-		 * If @prev is runnable & has slice left, it has priority and
-		 * fetching more just increases latency for the fetched tasks.
-		 * Tell put_prev_task_scx() to put @prev on local_dsq. If the
-		 * BPF scheduler wants to handle this explicitly, it should
-		 * implement ->cpu_released().
-		 *
-		 * See scx_ops_disable_workfn() for the explanation on the
-		 * disabling() test.
-		 *
-		 * When balancing a remote CPU for core-sched, there won't be a
-		 * following put_prev_task_scx() call and we don't own
-		 * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the
-		 * same conditions later and pick @rq->curr accordingly.
-		 */
-		if ((prev->scx->flags & SCX_TASK_QUEUED) &&
-		    prev->scx->slice && !scx_ops_disabling()) {
-			if (local)
-				prev->scx->flags |= SCX_TASK_BAL_KEEP;
-			return 1;
-		}
-	}
-
-	/* if there already are tasks to run, nothing to do */
-	if (scx_rq->local_dsq.nr)
-		return 1;
-
-	if (consume_dispatch_q(rq, rf, &scx_dsq_global))
-		return 1;
-
-	if (!SCX_HAS_OP(dispatch))
-		return 0;
-
-	dspc->rq = rq;
-	dspc->rf = rf;
-
-	/*
-	 * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock,
-	 * the local DSQ might still end up empty after a successful
-	 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
-	 * produced some tasks, retry. The BPF scheduler may depend on this
-	 * looping behavior to simplify its implementation.
-	 */
-	do {
-		dspc->nr_tasks = 0;
-
-		SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq),
-			    prev_on_scx ? prev : NULL);
-
-		flush_dispatch_buf(rq, rf);
-
-		if (scx_rq->local_dsq.nr)
-			return 1;
-		if (consume_dispatch_q(rq, rf, &scx_dsq_global))
-			return 1;
-
-		/*
-		 * ops.dispatch() can trap us in this loop by repeatedly
-		 * dispatching ineligible tasks. Break out once in a while to
-		 * allow the watchdog to run. As IRQ can't be enabled in
-		 * balance(), we want to complete this scheduling cycle and then
-		 * start a new one. IOW, we want to call resched_curr() on the
-		 * next, most likely idle, task, not the current one. Use
-		 * scx_bpf_kick_cpu() for deferred kicking.
-		 */
-		if (unlikely(!--nr_loops)) {
-			scx_bpf_kick_cpu(cpu_of(rq), 0);
-			break;
-		}
-	} while (dspc->nr_tasks);
-
-	return 0;
-}
-
-static int balance_scx(struct rq *rq, struct task_struct *prev,
-		       struct rq_flags *rf)
-{
-	int ret;
-
-	ret = balance_one(rq, prev, rf, true);
-#ifdef CONFIG_SCHED_SMT
-	/*
-	 * When core-sched is enabled, this ops.balance() call will be followed
-	 * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx()
-	 * on the SMT siblings. Balance the siblings too.
-	 */
-	if (sched_core_enabled(rq)) {
-		const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq));
-		int scpu;
-
-		for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) {
-			struct rq *srq = cpu_rq(scpu);
-			struct rq_flags srf;
-			struct task_struct *sprev = srq->curr;
-
-			/*
-			 * While core-scheduling, rq lock is shared among
-			 * siblings but the debug annotations and rq clock
-			 * aren't. Do pinning dance to transfer the ownership.
-			 */
-			WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq));
-			rq_unpin_lock(rq, rf);
-			rq_pin_lock(srq, &srf);
-
-			update_rq_clock(srq);
-			balance_one(srq, sprev, &srf, false);
-
-			rq_unpin_lock(srq, &srf);
-			rq_repin_lock(rq, rf);
-		}
-	}
-#endif
-	return ret;
-}
-
-static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
-{
-	if (p->scx->flags & SCX_TASK_QUEUED) {
-		/*
-		 * Core-sched might decide to execute @p before it is
-		 * dispatched. Call ops_dequeue() to notify the BPF scheduler.
-		 */
-		ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC);
-		dispatch_dequeue(rq->scx, p);
-	}
-
-	p->se.exec_start = rq_clock_task(rq);
-
-	/* see dequeue_task_scx() on why we skip when !QUEUED */
-	if (SCX_HAS_OP(running) && (p->scx->flags & SCX_TASK_QUEUED))
-		SCX_CALL_OP_TASK(SCX_KF_REST, running, p);
-
-	//if(p->scx->flags & SCX_TASK_QUEUED)
-	//	scx_update_task_ravg(p, rq, PICK_NEXT_TASK, rq->clock);
-
-	watchdog_unwatch_task(p, true);
-
-	/*
-	 * @p is getting newly scheduled or got kicked after someone updated its
-	 * slice. Refresh whether tick can be stopped. See can_stop_tick_scx().
-	 */
-	if ((p->scx->slice == SCX_SLICE_INF) !=
-	    (bool)(rq->scx->flags & SCX_RQ_CAN_STOP_TICK)) {
-		if (p->scx->slice == SCX_SLICE_INF)
-			rq->scx->flags |= SCX_RQ_CAN_STOP_TICK;
-		else
-			rq->scx->flags &= ~SCX_RQ_CAN_STOP_TICK;
-
-		sched_update_tick_dependency(rq);
-	}
-}
-
-static void put_prev_task_scx(struct rq *rq, struct task_struct *p)
-{
-#ifndef CONFIG_SMP
-	/*
-	 * UP workaround.
-	 *
-	 * Because SCX may transfer tasks across CPUs during dispatch, dispatch
-	 * is performed from its balance operation which isn't called in UP.
-	 * Let's work around by calling it from the operations which come right
-	 * after.
-	 *
-	 * 1. If the prev task is on SCX, pick_next_task() calls
-	 *    .put_prev_task() right after. As .put_prev_task() is also called
-	 *    from other places, we need to distinguish the calls which can be
-	 *    done by looking at the previous task's state - if still queued or
-	 *    dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task().
-	 *    This case is handled here.
-	 *
-	 * 2. If the prev task is not on SCX, the first following call into SCX
-	 *    will be .pick_next_task(), which is covered by calling
-	 *    balance_scx() from pick_next_task_scx().
-	 *
-	 * Note that we can't merge the first case into the second as
-	 * balance_scx() must be called before the previous SCX task goes
-	 * through put_prev_task_scx().
-	 *
-	 * As UP doesn't transfer tasks around, balance_scx() doesn't need @rf.
-	 * Pass in %NULL.
-	 */
-	if (p->scx->flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP))
-		balance_scx(rq, p, NULL);
-#endif
-
-	update_curr_scx(rq);
-
-	/* see dequeue_task_scx() on why we skip when !QUEUED */
-	if (SCX_HAS_OP(stopping) && (p->scx->flags & SCX_TASK_QUEUED))
-		SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true);
-
-	//if(p->scx->flags & SCX_TASK_QUEUED)
-	//	scx_update_task_ravg(p, rq, PUT_PREV_TASK, rq->clock);
-
-
-	/*
-	 * If we're being called from put_prev_task_balance(), balance_scx() may
-	 * have decided that @p should keep running.
-	 */
-	if (p->scx->flags & SCX_TASK_BAL_KEEP) {
-		p->scx->flags &= ~SCX_TASK_BAL_KEEP;
-		watchdog_watch_task(rq, p);
-		dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD);
-		return;
-	}
-
-	if (p->scx->flags & SCX_TASK_QUEUED) {
-		watchdog_watch_task(rq, p);
-
-		/*
-		 * If @p has slice left and balance_scx() didn't tag it for
-		 * keeping, @p is getting preempted by a higher priority
-		 * scheduler class or core-sched forcing a different task. Leave
-		 * it at the head of the local DSQ.
-		 */
-		if (p->scx->slice && !scx_ops_disabling()) {
-			dispatch_enqueue(&rq->scx->local_dsq, p, SCX_ENQ_HEAD);
-			return;
-		}
-
-		/*
-		 * If we're in the pick_next_task path, balance_scx() should
-		 * have already populated the local DSQ if there are any other
-		 * available tasks. If empty, tell ops.enqueue() that @p is the
-		 * only one available for this cpu. ops.enqueue() should put it
-		 * on the local DSQ so that the subsequent pick_next_task_scx()
-		 * can find the task unless it wants to trigger a separate
-		 * follow-up scheduling event.
-		 */
-		if (list_empty(&rq->scx->local_dsq.fifo))
-			do_enqueue_task(rq, p, SCX_ENQ_LAST | SCX_ENQ_LOCAL, -1);
-		else
-			do_enqueue_task(rq, p, 0, -1);
-	}
-}
-
-static struct task_struct *first_local_task(struct rq *rq)
-{
-	struct rb_node *rb_node;
-	struct sched_ext_entity *entity;
-
-	if (!list_empty(&rq->scx->local_dsq.fifo)) {
-		entity = list_first_entry(&rq->scx->local_dsq.fifo, struct sched_ext_entity, dsq_node.fifo);
-		return entity->task;
-	}
-
-	rb_node = rb_first_cached(&rq->scx->local_dsq.priq);
-	if (rb_node) {
-		entity = container_of(rb_node, struct sched_ext_entity, dsq_node.priq);
-		return entity->task;
-	}
-
-	return NULL;
-}
-
-static struct task_struct *pick_next_task_scx(struct rq *rq)
-{
-	struct task_struct *p;
-
-#ifndef CONFIG_SMP
-	/* UP workaround - see the comment at the head of put_prev_task_scx() */
-	if (unlikely(rq->curr->sched_class != &ext_sched_class))
-		balance_scx(rq, rq->curr, NULL);
-#endif
-
-	p = first_local_task(rq);
-	if (!p)
-		return NULL;
-
-	if (unlikely(!p->scx->slice)) {
-		if (!scx_ops_disabling() && !scx_warned_zero_slice) {
-			printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n",
-					p->comm, p->pid);
-			scx_warned_zero_slice = true;
-		}
-		p->scx->slice = SCX_SLICE_DFL;
-	}
-
-	set_next_task_scx(rq, p, true);
-
-	return p;
-}
-
-#ifdef CONFIG_SCHED_CORE
-/**
- * scx_prio_less - Task ordering for core-sched
- * @a: task A
- * @b: task B
- *
- * Core-sched is implemented as an additional scheduling layer on top of the
- * usual sched_class'es and needs to find out the expected task ordering. For
- * SCX, core-sched calls this function to interrogate the task ordering.
- *
- * Unless overridden by ops.core_sched_before(), @p->scx->core_sched_at is used
- * to implement the default task ordering. The older the timestamp, the higher
- * prority the task - the global FIFO ordering matching the default scheduling
- * behavior.
- *
- * When ops.core_sched_before() is enabled, @p->scx->core_sched_at is used to
- * implement FIFO ordering within each local DSQ. See pick_task_scx().
- */
-bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
-		   bool in_fi)
-{
-	/*
-	 * The const qualifiers are dropped from task_struct pointers when
-	 * calling ops.core_sched_before(). Accesses are controlled by the
-	 * verifier.
-	 */
-	if (SCX_HAS_OP(core_sched_before) && !scx_ops_disabling())
-		return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before,
-					      (struct task_struct *)a,
-					      (struct task_struct *)b);
-	else
-		return time_after64(a->scx->core_sched_at, b->scx->core_sched_at);
-}
-
-/**
- * pick_task_scx - Pick a candidate task for core-sched
- * @rq: rq to pick the candidate task from
- *
- * Core-sched calls this function on each SMT sibling to determine the next
- * tasks to run on the SMT siblings. balance_one() has been called on all
- * siblings and put_prev_task_scx() has been called only for the current CPU.
- *
- * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look
- * at the first task in the local dsq. @rq->curr has to be considered explicitly
- * to mimic %SCX_TASK_BAL_KEEP.
- */
-static struct task_struct *pick_task_scx(struct rq *rq)
-{
-	struct task_struct *curr = rq->curr;
-	struct task_struct *first = first_local_task(rq);
-
-	if (curr->scx->flags & SCX_TASK_QUEUED) {
-		/* is curr the only runnable task? */
-		if (!first)
-			return curr;
-
-		/*
-		 * Does curr trump first? We can always go by core_sched_at for
-		 * this comparison as it represents global FIFO ordering when
-		 * the default core-sched ordering is used and local-DSQ FIFO
-		 * ordering otherwise.
-		 *
-		 * We can have a task with an earlier timestamp on the DSQ. For
-		 * example, when a current task is preempted by a sibling
-		 * picking a different cookie, the task would be requeued at the
-		 * head of the local DSQ with an earlier timestamp than the
-		 * core-sched picked next task. Besides, the BPF scheduler may
-		 * dispatch any tasks to the local DSQ anytime.
-		 */
-		if (curr->scx->slice && time_before64(curr->scx->core_sched_at,
-						     first->scx->core_sched_at))
-			return curr;
-	}
-
-	return first;	/* this may be %NULL */
-}
-#endif	/* CONFIG_SCHED_CORE */
-
-static enum scx_cpu_preempt_reason
-preempt_reason_from_class(const struct sched_class *class)
-{
-#ifdef CONFIG_SMP
-	if (class == &stop_sched_class)
-		return SCX_CPU_PREEMPT_STOP;
-#endif
-	if (class == &dl_sched_class)
-		return SCX_CPU_PREEMPT_DL;
-	if (class == &rt_sched_class)
-		return SCX_CPU_PREEMPT_RT;
-	return SCX_CPU_PREEMPT_UNKNOWN;
-}
-
-void __scx_notify_pick_next_task(struct rq *rq, struct task_struct *task,
-				 const struct sched_class *active)
-{
-	lockdep_assert_rq_held(rq);
-
-	/*
-	 * The callback is conceptually meant to convey that the CPU is no
-	 * longer under the control of SCX. Therefore, don't invoke the
-	 * callback if the CPU is is staying on SCX, or going idle (in which
-	 * case the SCX scheduler has actively decided not to schedule any
-	 * tasks on the CPU).
-	 */
-	if (likely(active >= &ext_sched_class))
-		return;
-
-	/*
-	 * At this point we know that SCX was preempted by a higher priority
-	 * sched_class, so invoke the ->cpu_release() callback if we have not
-	 * done so already. We only send the callback once between SCX being
-	 * preempted, and it regaining control of the CPU.
-	 *
-	 * ->cpu_release() complements ->cpu_acquire(), which is emitted the
-	 *  next time that balance_scx() is invoked.
-	 */
-	if (!rq->scx->cpu_released) {
-		if (SCX_HAS_OP(cpu_release)) {
-			struct scx_cpu_release_args args = {
-				.reason = preempt_reason_from_class(active),
-				.task = task,
-			};
-
-			SCX_CALL_OP(SCX_KF_CPU_RELEASE,
-				    cpu_release, cpu_of(rq), &args);
-		}
-		rq->scx->cpu_released = true;
-	}
-}
-
-#ifdef CONFIG_SMP
-
-static bool test_and_clear_cpu_idle(int cpu)
-{
-	if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) {
-		if (cpumask_empty(idle_masks.cpu))
-			scx_has_idle_cpus = false;
-		return true;
-	} else {
-		return false;
-	}
-}
-
-static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed)
-{
-	int cpu;
-
-	do {
-		cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed);
-		if (cpu < nr_cpu_ids) {
-			const struct cpumask *sbm = topology_sibling_cpumask(cpu);
-
-			/*
-			 * If offline, @cpu is not its own sibling and we can
-			 * get caught in an infinite loop as @cpu is never
-			 * cleared from idle_masks.smt. Clear @cpu directly in
-			 * such cases.
-			 */
-			if (likely(cpumask_test_cpu(cpu, sbm)))
-				cpumask_andnot(idle_masks.smt, idle_masks.smt, sbm);
-			else
-				cpumask_andnot(idle_masks.smt, idle_masks.smt, cpumask_of(cpu));
-		} else {
-			cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed);
-			if (cpu >= nr_cpu_ids)
-				return -EBUSY;
-		}
-	} while (!test_and_clear_cpu_idle(cpu));
-
-	return cpu;
-}
-
-static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags)
-{
-	s32 cpu;
-
-	if (!static_branch_likely(&scx_builtin_idle_enabled)) {
-		scx_ops_error("built-in idle tracking is disabled");
-		return prev_cpu;
-	}
-
-	/*
-	 * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the
-	 * local DSQ of the waker.
-	 */
-	if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 &&
-	    scx_has_idle_cpus && !(current->flags & PF_EXITING)) {
-		cpu = smp_processor_id();
-		if (cpumask_test_cpu(cpu, p->cpus_ptr)) {
-			p->scx->flags |= SCX_TASK_ENQ_LOCAL;
-			return cpu;
-		}
-	}
-
-	/* if the previous CPU is idle, dispatch directly to it */
-	if (test_and_clear_cpu_idle(prev_cpu)) {
-		p->scx->flags |= SCX_TASK_ENQ_LOCAL;
-		return prev_cpu;
-	}
-
-	if (p->nr_cpus_allowed == 1)
-		return prev_cpu;
-
-	cpu = scx_pick_idle_cpu(p->cpus_ptr);
-	if (cpu >= 0) {
-		p->scx->flags |= SCX_TASK_ENQ_LOCAL;
-		return cpu;
-	}
-
-	return prev_cpu;
-}
-
-static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)
-{
-	if (SCX_HAS_OP(select_cpu)) {
-		s32 cpu;
-
-		cpu = SCX_CALL_OP_TASK_RET(SCX_KF_REST, select_cpu, p, prev_cpu,
-					   wake_flags);
-		if (ops_cpu_valid(cpu)) {
-			return cpu;
-		} else {
-			scx_ops_error("select_cpu returned invalid cpu %d", cpu);
-			return prev_cpu;
-		}
-	} else {
-		return scx_select_cpu_dfl(p, prev_cpu, wake_flags);
-	}
-}
-
-static void set_cpus_allowed_scx(struct task_struct *p, struct affinity_context *ctx)
-{
-	set_cpus_allowed_common(p, ctx);
-
-	/*
-	 * The effective cpumask is stored in @p->cpus_ptr which may temporarily
-	 * differ from the configured one in @p->cpus_mask. Always tell the bpf
-	 * scheduler the effective one.
-	 *
-	 * Fine-grained memory write control is enforced by BPF making the const
-	 * designation pointless. Cast it away when calling the operation.
-	 */
-	if (SCX_HAS_OP(set_cpumask))
-		SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p,
-				 (struct cpumask *)p->cpus_ptr);
-}
-
-static void reset_idle_masks(void)
-{
-	/* consider all cpus idle, should converge to the actual state quickly */
-	cpumask_setall(idle_masks.cpu);
-	cpumask_setall(idle_masks.smt);
-	scx_has_idle_cpus = true;
-}
-
-void __scx_update_idle(struct rq *rq, bool idle)
-{
-	int cpu = cpu_of(rq);
-	struct cpumask *sib_mask = topology_sibling_cpumask(cpu);
-
-	if (SCX_HAS_OP(update_idle)) {
-		SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle);
-		if (!static_branch_unlikely(&scx_builtin_idle_enabled))
-			return;
-	}
-
-	if (idle) {
-		cpumask_set_cpu(cpu, idle_masks.cpu);
-		if (!scx_has_idle_cpus)
-			scx_has_idle_cpus = true;
-
-		/*
-		 * idle_masks.smt handling is racy but that's fine as it's only
-		 * for optimization and self-correcting.
-		 */
-		for_each_cpu(cpu, sib_mask) {
-			if (!cpumask_test_cpu(cpu, idle_masks.cpu))
-				return;
-		}
-		cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask);
-	} else {
-		cpumask_clear_cpu(cpu, idle_masks.cpu);
-		if (scx_has_idle_cpus && cpumask_empty(idle_masks.cpu))
-			scx_has_idle_cpus = false;
-
-		cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask);
-	}
-}
-
-#else /* !CONFIG_SMP */
-
-static bool test_and_clear_cpu_idle(int cpu) { return false; }
-static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; }
-static void reset_idle_masks(void) {}
-
-#endif /* CONFIG_SMP */
-
-static bool check_rq_for_timeouts(struct rq *rq)
-{
-	struct sched_ext_entity *entity;
-	struct task_struct *p;
-	struct rq_flags rf;
-	bool timed_out = false;
-
-	rq_lock_irqsave(rq, &rf);
-	list_for_each_entry(entity, &rq->scx->watchdog_list, watchdog_node) {
-		unsigned long last_runnable;
-
-		p = entity->task;
-		last_runnable = p->scx->runnable_at;
-
-		if (unlikely(time_after(jiffies,
-					last_runnable + scx_watchdog_timeout))) {
-			u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
-
-			scx_ops_error_type(SCX_EXIT_ERROR_STALL,
-					   "%s[%d] failed to run for %u.%03us",
-					   p->comm, p->pid,
-					   dur_ms / 1000, dur_ms % 1000);
-			timed_out = true;
-			break;
-		}
-	}
-	rq_unlock_irqrestore(rq, &rf);
-
-	return timed_out;
-}
-
-static void scx_watchdog_workfn(struct work_struct *work)
-{
-	int cpu;
-
-	scx_watchdog_timestamp = jiffies;
-
-	for_each_online_cpu(cpu) {
-		if (unlikely(check_rq_for_timeouts(cpu_rq(cpu))))
-			break;
-
-		cond_resched();
-	}
-	queue_delayed_work(system_unbound_wq, to_delayed_work(work),
-			   scx_watchdog_timeout / 2);
-}
-
-static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
-{
-	update_curr_scx(rq);
-	//scx_update_task_ravg(curr, rq, TASK_UPDATE, rq->clock);
-	/*
-	 * While disabling, always resched and refresh core-sched timestamp as
-	 * we can't trust the slice management or ops.core_sched_before().
-	 */
-	if (scx_ops_disabling()) {
-		curr->scx->slice = 0;
-		touch_core_sched(rq, curr);
-	}
-
-	if (!curr->scx->slice)
-		resched_curr(rq);
-}
-
-#define SCX_ENABLE_ARGS_INIT_CGROUP(tg)
-
-static int scx_ops_prepare_task(struct task_struct *p, struct task_group *tg)
-{
-	int ret;
-
-	WARN_ON_ONCE(p->scx->flags & SCX_TASK_OPS_PREPPED);
-
-	p->scx->disallow = false;
-
-	if (SCX_HAS_OP(prep_enable)) {
-		struct scx_enable_args args = {
-			SCX_ENABLE_ARGS_INIT_CGROUP(tg)
-		};
-
-		ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, prep_enable, p, &args);
-		if (unlikely(ret)) {
-			ret = ops_sanitize_err("prep_enable", ret);
-			return ret;
-		}
-	}
-	//scx_sched_init_task(p);
-
-	if (p->scx->disallow) {
-		struct rq *rq;
-		struct rq_flags rf;
-
-		rq = task_rq_lock(p, &rf);
-
-		/*
-		 * We're either in fork or load path and @p->policy will be
-		 * applied right after. Reverting @p->policy here and rejecting
-		 * %SCHED_EXT transitions from scx_check_setscheduler()
-		 * guarantees that if ops.prep_enable() sets @p->disallow, @p
-		 * can never be in SCX.
-		 */
-		if (p->policy == SCHED_EXT) {
-			p->policy = SCHED_NORMAL;
-			atomic64_inc(&scx_nr_rejected);
-		}
-
-		task_rq_unlock(rq, p, &rf);
-	}
-
-	p->scx->flags |= (SCX_TASK_OPS_PREPPED | SCX_TASK_WATCHDOG_RESET);
-	return 0;
-}
-
-static void scx_ops_enable_task(struct task_struct *p)
-{
-	lockdep_assert_rq_held(task_rq(p));
-	WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_OPS_PREPPED));
-
-	if (SCX_HAS_OP(enable)) {
-		struct scx_enable_args args = {
-			SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p))
-		};
-		SCX_CALL_OP_TASK(SCX_KF_REST, enable, p, &args);
-	}
-	p->scx->flags &= ~SCX_TASK_OPS_PREPPED;
-	p->scx->flags |= SCX_TASK_OPS_ENABLED;
-}
-
-static void scx_ops_disable_task(struct task_struct *p)
-{
-	lockdep_assert_rq_held(task_rq(p));
-
-	if (p->scx->flags & SCX_TASK_OPS_PREPPED) {
-		if (SCX_HAS_OP(cancel_enable)) {
-			struct scx_enable_args args = {
-				SCX_ENABLE_ARGS_INIT_CGROUP(task_group(p))
-			};
-			SCX_CALL_OP(SCX_KF_REST, cancel_enable, p, &args);
-		}
-		p->scx->flags &= ~SCX_TASK_OPS_PREPPED;
-	} else if (p->scx->flags & SCX_TASK_OPS_ENABLED) {
-		if (SCX_HAS_OP(disable))
-			SCX_CALL_OP(SCX_KF_REST, disable, p);
-		p->scx->flags &= ~SCX_TASK_OPS_ENABLED;
-	}
-}
-
-static void set_task_scx_weight(struct task_struct *p)
-{
-	u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
-
-	p->scx->weight = sched_weight_to_cgroup(weight);
-}
-
-/**
- * refresh_scx_weight - Refresh a task's ext weight
- * @p: task to refresh ext weight for
- *
- * @p->scx->weight carries the task's static priority in cgroup weight scale to
- * enable easy access from the BPF scheduler. To keep it synchronized with the
- * current task priority, this function should be called when a new task is
- * created, priority is changed for a task on sched_ext, and a task is switched
- * to sched_ext from other classes.
- */
-static void refresh_scx_weight(struct task_struct *p)
-{
-	lockdep_assert_rq_held(task_rq(p));
-	set_task_scx_weight(p);
-	if (SCX_HAS_OP(set_weight))
-		SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx->weight);
-}
-
-void scx_pre_fork(struct task_struct *p)
-{
-	p->scx = kmalloc(sizeof(struct sched_ext_entity), GFP_KERNEL);
-	if (!p->scx) {
-		goto lock;
-	}
-
-	p->scx->dsq = NULL;
-	INIT_LIST_HEAD(&p->scx->dsq_node.fifo);
-	RB_CLEAR_NODE(&p->scx->dsq_node.priq);
-	INIT_LIST_HEAD(&p->scx->watchdog_node);
-	p->scx->flags = 0;
-	p->scx->weight = 0;
-	p->scx->sticky_cpu = -1;
-	p->scx->holding_cpu = -1;
-	p->scx->kf_mask = 0;
-	atomic64_set(&p->scx->ops_state, 0);
-	p->scx->runnable_at = INITIAL_JIFFIES;
-	p->scx->slice = SCX_SLICE_DFL;
-	p->scx->task = p;
-	p->sched_prop = 0;
-
-	/*
-	 * BPF scheduler enable/disable paths want to be able to iterate and
-	 * update all tasks which can become complex when racing forks. As
-	 * enable/disable are very cold paths, let's use a percpu_rwsem to
-	 * exclude forks.
-	 */
-lock:
-	percpu_down_read(&scx_fork_rwsem);
-}
-
-int scx_fork(struct task_struct *p)
-{
-	percpu_rwsem_assert_held(&scx_fork_rwsem);
-
-	if (scx_enabled())
-		return scx_ops_prepare_task(p, task_group(p));
-	else
-		return 0;
-}
-
-void scx_post_fork(struct task_struct *p)
-{
-	if (scx_enabled()) {
-		struct rq_flags rf;
-		struct rq *rq;
-
-		rq = task_rq_lock(p, &rf);
-		/*
-		 * Set the weight manually before calling ops.enable() so that
-		 * the scheduler doesn't see a stale value if they inspect the
-		 * task struct. We'll invoke ops.set_weight() afterwards, as it
-		 * would be odd to receive a callback on the task before we
-		 * tell the scheduler that it's been fully enabled.
-		 */
-		set_task_scx_weight(p);
-		scx_ops_enable_task(p);
-		refresh_scx_weight(p);
-		task_rq_unlock(rq, p, &rf);
-	}
-
-	spin_lock_irq(&scx_tasks_lock);
-	list_add_tail(&p->scx->tasks_node, &scx_tasks);
-	spin_unlock_irq(&scx_tasks_lock);
-
-	percpu_up_read(&scx_fork_rwsem);
-}
-
-void scx_cancel_fork(struct task_struct *p)
-{
-	if (scx_enabled())
-		scx_ops_disable_task(p);
-	percpu_up_read(&scx_fork_rwsem);
-}
-
-void sched_ext_free(struct task_struct *p)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&scx_tasks_lock, flags);
-	list_del_init(&p->scx->tasks_node);
-	spin_unlock_irqrestore(&scx_tasks_lock, flags);
-
-	/*
-	 * @p is off scx_tasks and wholly ours. scx_ops_enable()'s PREPPED ->
-	 * ENABLED transitions can't race us. Disable ops for @p.
-	 */
-	if (p->scx->flags & (SCX_TASK_OPS_PREPPED | SCX_TASK_OPS_ENABLED)) {
-		struct rq_flags rf;
-		struct rq *rq;
-
-		rq = task_rq_lock(p, &rf);
-		scx_ops_disable_task(p);
-		task_rq_unlock(rq, p, &rf);
-	}
-}
-
-static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio)
-{
-}
-
-static void check_preempt_curr_scx(struct rq *rq, struct task_struct *p,int wake_flags) {}
-static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
-
-int scx_check_setscheduler(struct task_struct *p, int policy)
-{
-	lockdep_assert_rq_held(task_rq(p));
-
-	/* if disallow, reject transitioning into SCX */
-	if (scx_enabled() && READ_ONCE(p->scx->disallow) &&
-	    p->policy != policy && policy == SCHED_EXT)
-		return -EACCES;
-
-	return 0;
-}
-
-#ifdef CONFIG_NO_HZ_FULL
-bool scx_can_stop_tick(struct rq *rq)
-{
-	struct task_struct *p = rq->curr;
-
-	if (scx_ops_disabling())
-		return false;
-
-	if (p->sched_class != &ext_sched_class)
-		return true;
-
-	/*
-	 * @rq can dispatch from different DSQs, so we can't tell whether it
-	 * needs the tick or not by looking at nr_running. Allow stopping ticks
-	 * iff the BPF scheduler indicated so. See set_next_task_scx().
-	 */
-	return rq->scx->flags & SCX_RQ_CAN_STOP_TICK;
-}
-#endif
-
-static inline void scx_cgroup_lock(void) {}
-static inline void scx_cgroup_unlock(void) {}
-
-/*
- * Omitted operations:
- *
- * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the
- *   task isn't tied to the CPU at that point. Preemption is implemented by
- *   resetting the victim task's slice to 0 and triggering reschedule on the
- *   target CPU.
- *
- * - migrate_task_rq: Unncessary as task to cpu mapping is transient.
- *
- * - task_fork/dead: We need fork/dead notifications for all tasks regardless of
- *   their current sched_class. Call them directly from sched core instead.
- *
- * - task_woken, switched_from: Unnecessary.
- */
-DEFINE_SCHED_CLASS(ext) = {
-	.enqueue_task		= enqueue_task_scx,
-	.dequeue_task		= dequeue_task_scx,
-	.yield_task		= yield_task_scx,
-	.yield_to_task		= yield_to_task_scx,
-
-	.check_preempt_curr	= check_preempt_curr_scx,
-
-	.pick_next_task		= pick_next_task_scx,
-
-	.put_prev_task		= put_prev_task_scx,
-	.set_next_task          = set_next_task_scx,
-
-#ifdef CONFIG_SMP
-	.balance		= balance_scx,
-	.select_task_rq		= select_task_rq_scx,
-	.set_cpus_allowed	= set_cpus_allowed_scx,
-#endif
-
-#ifdef CONFIG_SCHED_CORE
-	.pick_task		= pick_task_scx,
-#endif
-
-	.task_tick		= task_tick_scx,
-
-	.switched_to		= switched_to_scx,
-	.prio_changed		= prio_changed_scx,
-
-	.update_curr		= update_curr_scx,
-
-#ifdef CONFIG_UCLAMP_TASK
-	.uclamp_enabled		= 0,
-#endif
-};
-
-static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id)
-{
-	memset(dsq, 0, sizeof(*dsq));
-
-	raw_spin_lock_init(&dsq->lock);
-	INIT_LIST_HEAD(&dsq->fifo);
-	dsq->id = dsq_id;
-}
-
-static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node)
-{
-	struct scx_dispatch_q *dsq;
-
-	if (dsq_id & SCX_DSQ_FLAG_BUILTIN)
-		return ERR_PTR(-EINVAL);
-
-	dsq = kmalloc_node(sizeof(*dsq), GFP_ATOMIC, node);
-	if (!dsq)
-		return ERR_PTR(-ENOMEM);
-
-	init_dsq(dsq, dsq_id);
-
-	return dsq;
-}
-
-static void free_dsq_irq_workfn(struct irq_work *irq_work)
-{
-	struct llist_node *to_free = llist_del_all(&dsqs_to_free);
-	struct scx_dispatch_q *dsq, *tmp_dsq;
-
-	llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node)
-		kfree_rcu(dsq, rcu);
-}
-
-static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
-
-static void destroy_dsq(u64 dsq_id)
-{
-}
-
-static void scx_cgroup_exit(void) {}
-static int scx_cgroup_init(void) { return 0; }
-static void scx_cgroup_config_knobs(void) {}
-
-/*
- * Used by sched_fork() and __setscheduler_prio() to pick the matching
- * sched_class. dl/rt are already handled.
- */
-bool task_on_scx(struct task_struct *p)
-{
-	if (!scx_enabled() || scx_ops_disabling())
-		return false;
-	if (READ_ONCE(scx_switching_all))
-		return true;
-	return p->policy == SCHED_EXT;
-}
-
-static void scx_ops_fallback_enqueue(struct task_struct *p, u64 enq_flags)
-{
-	if (enq_flags & SCX_ENQ_LAST)
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
-	else
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
-}
-
-static void scx_ops_fallback_dispatch(s32 cpu, struct task_struct *prev) {}
-
-static void scx_ops_disable_workfn(struct kthread_work *work)
-{
-	struct scx_exit_info *ei = &scx_exit_info;
-	struct scx_task_iter sti;
-	struct task_struct *p;
-	const char *reason;
-	int i, cpu, type;
-
-	type = atomic_read(&scx_exit_type);
-	while (true) {
-		/*
-		 * NONE indicates that a new scx_ops has been registered since
-		 * disable was scheduled - don't kill the new ops. DONE
-		 * indicates that the ops has already been disabled.
-		 */
-		if (type == SCX_EXIT_NONE || type == SCX_EXIT_DONE)
-			return;
-		if (atomic_try_cmpxchg(&scx_exit_type, &type, SCX_EXIT_DONE))
-			break;
-	}
-
-	cancel_delayed_work_sync(&scx_watchdog_work);
-
-	switch (type) {
-	case SCX_EXIT_UNREG:
-		reason = "BPF scheduler unregistered";
-		break;
-	case SCX_EXIT_SYSRQ:
-		reason = "disabled by sysrq-S";
-		break;
-	case SCX_EXIT_ERROR:
-		reason = "runtime error";
-		break;
-	case SCX_EXIT_ERROR_BPF:
-		reason = "scx_bpf_error";
-		break;
-	case SCX_EXIT_ERROR_STALL:
-		reason = "runnable task stall";
-		break;
-	default:
-		reason = "<UNKNOWN>";
-	}
-
-	ei->type = type;
-	strlcpy(ei->reason, reason, sizeof(ei->reason));
-
-	switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) {
-	case SCX_OPS_DISABLED:
-		pr_warn("sched_ext: ops error detected without ops (%s)\n",
-			scx_exit_info.msg);
-		WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) !=
-			     SCX_OPS_DISABLING);
-		return;
-	case SCX_OPS_PREPPING:
-		goto forward_progress_guaranteed;
-	case SCX_OPS_DISABLING:
-		/* shouldn't happen but handle it like ENABLING if it does */
-		WARN_ONCE(true, "sched_ext: duplicate disabling instance?");
-		fallthrough;
-	case SCX_OPS_ENABLING:
-	case SCX_OPS_ENABLED:
-		break;
-	}
-
-	/*
-	 * DISABLING is set and ops was either ENABLING or ENABLED indicating
-	 * that the ops and static branches are set.
-	 *
-	 * We must guarantee that all runnable tasks make forward progress
-	 * without trusting the BPF scheduler. We can't grab any mutexes or
-	 * rwsems as they might be held by tasks that the BPF scheduler is
-	 * forgetting to run, which unfortunately also excludes toggling the
-	 * static branches.
-	 *
-	 * Let's work around by overriding a couple ops and modifying behaviors
-	 * based on the DISABLING state and then cycling the tasks through
-	 * dequeue/enqueue to force global FIFO scheduling.
-	 *
-	 * a. ops.enqueue() and .dispatch() are overridden for simple global
-	 *    FIFO scheduling.
-	 *
-	 * b. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value
-	 *    can't be trusted. Whenever a tick triggers, the running task is
-	 *    rotated to the tail of the queue with core_sched_at touched.
-	 *
-	 * c. pick_next_task() suppresses zero slice warning.
-	 *
-	 * d. scx_prio_less() reverts to the default core_sched_at order.
-	 */
-	scx_ops.enqueue = scx_ops_fallback_enqueue;
-	scx_ops.dispatch = scx_ops_fallback_dispatch;
-
-	spin_lock_irq(&scx_tasks_lock);
-	scx_task_iter_init(&sti);
-	while ((p = scx_task_iter_next_filtered_locked(&sti))) {
-		if (READ_ONCE(p->__state) != TASK_DEAD) {
-			SCHED_CHANGE_BLOCK(task_rq(p), p,
-					   DEQUEUE_SAVE | DEQUEUE_MOVE) {
-				/* cycling deq/enq is enough, see above */
-			}
-		}
-	}
-	scx_task_iter_exit(&sti);
-	spin_unlock_irq(&scx_tasks_lock);
-
-	/* kick all CPUs to restore ticks */
-	for_each_possible_cpu(cpu)
-		resched_cpu(cpu);
-
-forward_progress_guaranteed:
-	/*
-	 * Here, every runnable task is guaranteed to make forward progress and
-	 * we can safely use blocking synchronization constructs. Actually
-	 * disable ops.
-	 */
-	mutex_lock(&scx_ops_enable_mutex);
-
-	static_branch_disable(&__scx_switched_all);
-	WRITE_ONCE(scx_switching_all, false);
-
-	/* avoid racing against fork and cgroup changes */
-	cpus_read_lock();
-	percpu_down_write(&scx_fork_rwsem);
-	scx_cgroup_lock();
-
-	spin_lock_irq(&scx_tasks_lock);
-	scx_task_iter_init(&sti);
-	while ((p = scx_task_iter_next_filtered_locked(&sti))) {
-		const struct sched_class *old_class = p->sched_class;
-		struct rq *rq = task_rq(p);
-		bool alive = READ_ONCE(p->__state) != TASK_DEAD;
-
-		update_rq_clock(rq);
-
-		SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE |
-				   DEQUEUE_NOCLOCK) {
-			p->scx->slice = min_t(u64, p->scx->slice, SCX_SLICE_DFL);
-
-			__setscheduler_prio(p, p->prio);
-		}
-
-		if (alive)
-			check_class_changed(task_rq(p), p, old_class, p->prio);
-
-		scx_ops_disable_task(p);
-	}
-	scx_task_iter_exit(&sti);
-	spin_unlock_irq(&scx_tasks_lock);
-
-	/* no task is on scx, turn off all the switches and flush in-progress calls */
-	static_branch_disable_cpuslocked(&__scx_ops_enabled);
-	for (i = 0; i < SCX_NR_ONLINE_OPS; i++)
-		static_branch_disable_cpuslocked(&scx_has_op[i]);
-	static_branch_disable_cpuslocked(&scx_ops_enq_last);
-	static_branch_disable_cpuslocked(&scx_ops_enq_exiting);
-	static_branch_disable_cpuslocked(&scx_ops_cpu_preempt);
-	static_branch_disable_cpuslocked(&scx_builtin_idle_enabled);
-	synchronize_rcu();
-
-	scx_cgroup_exit();
-
-	scx_cgroup_unlock();
-	percpu_up_write(&scx_fork_rwsem);
-	cpus_read_unlock();
-
-	if (ei->type >= SCX_EXIT_ERROR) {
-		printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name);
-
-		if (ei->msg[0] == '\0')
-			printk(KERN_ERR "sched_ext: %s\n", ei->reason);
-		else
-			printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg);
-
-		stack_trace_print(ei->bt, ei->bt_len, 2);
-	}
-
-	if (scx_ops.exit)
-		SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei);
-
-	memset(&scx_ops, 0, sizeof(scx_ops));
-
-	free_percpu(scx_dsp_buf);
-	scx_dsp_buf = NULL;
-	scx_dsp_max_batch = 0;
-
-	mutex_unlock(&scx_ops_enable_mutex);
-
-	WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) !=
-		     SCX_OPS_DISABLING);
-
-	scx_cgroup_config_knobs();
-}
-
-static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn);
-
-static void schedule_scx_ops_disable_work(void)
-{
-	struct kthread_worker *helper = READ_ONCE(scx_ops_helper);
-
-	/*
-	 * We may be called spuriously before the first bpf_sched_ext_reg(). If
-	 * scx_ops_helper isn't set up yet, there's nothing to do.
-	 */
-	if (helper)
-		kthread_queue_work(helper, &scx_ops_disable_work);
-}
-
-static void scx_ops_disable(enum scx_exit_type type)
-{
-	int none = SCX_EXIT_NONE;
-
-	if (WARN_ON_ONCE(type == SCX_EXIT_NONE || type == SCX_EXIT_DONE))
-		type = SCX_EXIT_ERROR;
-
-	atomic_try_cmpxchg(&scx_exit_type, &none, type);
-
-	schedule_scx_ops_disable_work();
-}
-
-static void scx_ops_error_irq_workfn(struct irq_work *irq_work)
-{
-	schedule_scx_ops_disable_work();
-}
-
-static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn);
-
-__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type,
-				       const char *fmt, ...)
-{
-	struct scx_exit_info *ei = &scx_exit_info;
-	int none = SCX_EXIT_NONE;
-	va_list args;
-
-	if (!atomic_try_cmpxchg(&scx_exit_type, &none, type))
-		return;
-
-	ei->bt_len = stack_trace_save(ei->bt, ARRAY_SIZE(ei->bt), 1);
-
-	va_start(args, fmt);
-	vscnprintf(ei->msg, ARRAY_SIZE(ei->msg), fmt, args);
-	va_end(args);
-
-	irq_work_queue(&scx_ops_error_irq_work);
-}
-
-static struct kthread_worker *scx_create_rt_helper(const char *name)
-{
-	struct kthread_worker *helper;
-
-	helper = kthread_create_worker(0, name);
-	if (helper)
-		sched_set_fifo(helper->task);
-	return helper;
-}
-
-static int scx_ops_enable(struct sched_ext_ops *ops)
-{
-	struct scx_task_iter sti;
-	struct task_struct *p;
-	int i, ret;
-	int tcnt = 0;
-	unsigned long long start = 0;
-	unsigned long  long total_start = sched_clock();
-
-	mutex_lock(&scx_ops_enable_mutex);
-
-	if (!scx_ops_helper) {
-		WRITE_ONCE(scx_ops_helper,
-			   scx_create_rt_helper("sched_ext_ops_helper"));
-		if (!scx_ops_helper) {
-			ret = -ENOMEM;
-			goto err_unlock;
-		}
-	}
-
-	if (scx_ops_enable_state() != SCX_OPS_DISABLED) {
-		ret = -EBUSY;
-		goto err_unlock;
-	}
-
-	//slim_walt_enable(true);
-
-	/*
-	 * Set scx_ops, transition to PREPPING and clear exit info to arm the
-	 * disable path. Failure triggers full disabling from here on.
-	 */
-	scx_ops = *ops;
-
-	WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) !=
-		     SCX_OPS_DISABLED);
-
-	memset(&scx_exit_info, 0, sizeof(scx_exit_info));
-	atomic_set(&scx_exit_type, SCX_EXIT_NONE);
-	scx_warned_zero_slice = false;
-
-	atomic64_set(&scx_nr_rejected, 0);
-
-	/*
-	 * Keep CPUs stable during enable so that the BPF scheduler can track
-	 * online CPUs by watching ->on/offline_cpu() after ->init().
-	 */
-	cpus_read_lock();
-
-	scx_switch_all_req = true;
-	if (scx_ops.init) {
-		ret = SCX_CALL_OP_RET(SCX_KF_INIT, init);
-		if (ret) {
-			ret = ops_sanitize_err("init", ret);
-			goto err_disable;
-		}
-
-		/*
-		 * Exit early if ops.init() triggered scx_bpf_error(). Not
-		 * strictly necessary as we'll fail transitioning into ENABLING
-		 * later but that'd be after calling ops.prep_enable() on all
-		 * tasks and with -EBUSY which isn't very intuitive. Let's exit
-		 * early with success so that the condition is notified through
-		 * ops.exit() like other scx_bpf_error() invocations.
-		 */
-		if (atomic_read(&scx_exit_type) != SCX_EXIT_NONE)
-			goto err_disable;
-	}
-
-	WARN_ON_ONCE(scx_dsp_buf);
-	scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
-	scx_dsp_buf = __alloc_percpu(sizeof(scx_dsp_buf[0]) * scx_dsp_max_batch,
-				     __alignof__(scx_dsp_buf[0]));
-	if (!scx_dsp_buf) {
-		ret = -ENOMEM;
-		goto err_disable;
-	}
-
-	scx_watchdog_timeout = SCX_WATCHDOG_MAX_TIMEOUT;
-	if (ops->timeout_ms)
-		scx_watchdog_timeout = msecs_to_jiffies(ops->timeout_ms);
-
-	scx_watchdog_timestamp = jiffies;
-	queue_delayed_work(system_unbound_wq, &scx_watchdog_work,
-			   scx_watchdog_timeout / 2);
-
-	/*
-	 * Lock out forks, cgroup on/offlining and moves before opening the
-	 * floodgate so that they don't wander into the operations prematurely.
-	 */
-	percpu_down_write(&scx_fork_rwsem);
-	scx_cgroup_lock();
-
-	for (i = 0; i < SCX_NR_ONLINE_OPS; i++)
-		if (((void (**)(void))ops)[i])
-			static_branch_enable_cpuslocked(&scx_has_op[i]);
-
-	if (ops->flags & SCX_OPS_ENQ_LAST)
-		static_branch_enable_cpuslocked(&scx_ops_enq_last);
-
-	if (ops->flags & SCX_OPS_ENQ_EXITING)
-		static_branch_enable_cpuslocked(&scx_ops_enq_exiting);
-	if (scx_ops.cpu_acquire || scx_ops.cpu_release)
-		static_branch_enable_cpuslocked(&scx_ops_cpu_preempt);
-
-	if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) {
-		reset_idle_masks();
-		static_branch_enable_cpuslocked(&scx_builtin_idle_enabled);
-	} else {
-		static_branch_disable_cpuslocked(&scx_builtin_idle_enabled);
-	}
-
-	/*
-	 * All cgroups should be initialized before letting in tasks. cgroup
-	 * on/offlining and task migrations are already locked out.
-	 */
-	ret = scx_cgroup_init();
-	if (ret)
-		goto err_disable_unlock;
-
-	static_branch_enable_cpuslocked(&__scx_ops_enabled);
-
-	/*
-	 * Enable ops for every task. Fork is excluded by scx_fork_rwsem
-	 * preventing new tasks from being added. No need to exclude tasks
-	 * leaving as sched_ext_free() can handle both prepped and enabled
-	 * tasks. Prep all tasks first and then enable them with preemption
-	 * disabled.
-	 */
-	spin_lock_irq(&scx_tasks_lock);
-
-	scx_task_iter_init(&sti);
-	while ((p = scx_task_iter_next_filtered(&sti))) {
-		get_task_struct(p);
-		spin_unlock_irq(&scx_tasks_lock);
-
-		ret = scx_ops_prepare_task(p, task_group(p));
-		if (ret) {
-			put_task_struct(p);
-			spin_lock_irq(&scx_tasks_lock);
-			scx_task_iter_exit(&sti);
-			spin_unlock_irq(&scx_tasks_lock);
-			pr_err("sched_ext: ops.prep_enable() failed (%d) for %s[%d] while loading\n",
-			       ret, p->comm, p->pid);
-			goto err_disable_unlock;
-		}
-
-		put_task_struct(p);
-		spin_lock_irq(&scx_tasks_lock);
-	}
-	scx_task_iter_exit(&sti);
-
-	/*
-	 * All tasks are prepped but are still ops-disabled. Ensure that
-	 * %current can't be scheduled out and switch everyone.
-	 * preempt_disable() is necessary because we can't guarantee that
-	 * %current won't be starved if scheduled out while switching.
-	 */
-	preempt_disable();
-
-	/*
-	 * From here on, the disable path must assume that tasks have ops
-	 * enabled and need to be recovered.
-	 */
-	if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) {
-		preempt_enable();
-		spin_unlock_irq(&scx_tasks_lock);
-		ret = -EBUSY;
-		goto err_disable_unlock;
-	}
-
-	/*
-	 * We're fully committed and can't fail. The PREPPED -> ENABLED
-	 * transitions here are synchronized against sched_ext_free() through
-	 * scx_tasks_lock.
-	 */
-	WRITE_ONCE(scx_switching_all, scx_switch_all_req);
-	start = sched_clock();
-
-	scx_task_iter_init(&sti);
-	while ((p = scx_task_iter_next_filtered_locked(&sti))) {
-		tcnt++;
-		if (READ_ONCE(p->__state) != TASK_DEAD) {
-			const struct sched_class *old_class = p->sched_class;
-			struct rq *rq = task_rq(p);
-
-			update_rq_clock(rq);
-
-			SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE |
-					   DEQUEUE_NOCLOCK) {
-				scx_ops_enable_task(p);
-				__setscheduler_prio(p, p->prio);
-			}
-
-			check_class_changed(task_rq(p), p, old_class, p->prio);
-		} else {
-			scx_ops_disable_task(p);
-		}
-	}
-	printk("\n\n switch cnt = %d, duration = %llu, current cpu = %d \n\n", tcnt, sched_clock() - start, smp_processor_id());
-	scx_task_iter_exit(&sti);
-
-	spin_unlock_irq(&scx_tasks_lock);
-	preempt_enable();
-	scx_cgroup_unlock();
-	percpu_up_write(&scx_fork_rwsem);
-
-	if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) {
-		ret = -EBUSY;
-		goto err_disable;
-	}
-
-	if (scx_switch_all_req)
-		static_branch_enable_cpuslocked(&__scx_switched_all);
-
-	cpus_read_unlock();
-	mutex_unlock(&scx_ops_enable_mutex);
-
-	scx_cgroup_config_knobs();
-	printk("\n total duration = %llu , current cpu = %d\n",  sched_clock() - total_start, smp_processor_id());
-
-	return 0;
-
-err_unlock:
-	mutex_unlock(&scx_ops_enable_mutex);
-	return ret;
-
-err_disable_unlock:
-	scx_cgroup_unlock();
-	percpu_up_write(&scx_fork_rwsem);
-err_disable:
-	cpus_read_unlock();
-	mutex_unlock(&scx_ops_enable_mutex);
-	/* must be fully disabled before returning */
-	scx_ops_disable(SCX_EXIT_ERROR);
-	kthread_flush_work(&scx_ops_disable_work);
-	return ret;
-}
-
-#ifdef CONFIG_SCHED_DEBUG
-static const char *scx_ops_enable_state_str[] = {
-	[SCX_OPS_PREPPING]	= "prepping",
-	[SCX_OPS_ENABLING]	= "enabling",
-	[SCX_OPS_ENABLED]	= "enabled",
-	[SCX_OPS_DISABLING]	= "disabling",
-	[SCX_OPS_DISABLED]	= "disabled",
-};
-
-static int scx_debug_show(struct seq_file *m, void *v)
-{
-	mutex_lock(&scx_ops_enable_mutex);
-	seq_printf(m, "%-30s: %s\n", "ops", scx_ops.name);
-	seq_printf(m, "%-30s: %ld\n", "enabled", scx_enabled());
-	seq_printf(m, "%-30s: %d\n", "switching_all",
-		   READ_ONCE(scx_switching_all));
-	seq_printf(m, "%-30s: %ld\n", "switched_all", scx_switched_all());
-	seq_printf(m, "%-30s: %s\n", "enable_state",
-		   scx_ops_enable_state_str[scx_ops_enable_state()]);
-	seq_printf(m, "%-30s: %llu\n", "nr_rejected",
-		   atomic64_read(&scx_nr_rejected));
-	mutex_unlock(&scx_ops_enable_mutex);
-	return 0;
-}
-
-static int scx_debug_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, scx_debug_show, NULL);
-}
-
-const struct file_operations sched_ext_fops = {
-	.open		= scx_debug_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
-/********************************************************************************
- * bpf_struct_ops plumbing.
- */
-#include <linux/bpf_verifier.h>
-#include <linux/bpf.h>
-#include <linux/btf.h>
-
-extern struct btf *btf_vmlinux;
-static const struct btf_type *task_struct_type;
-
-static bool bpf_scx_is_valid_access(int off, int size,
-				    enum bpf_access_type type,
-				    const struct bpf_prog *prog,
-				    struct bpf_insn_access_aux *info)
-{
-	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
-		return false;
-	if (type != BPF_READ)
-		return false;
-	if (off % size != 0)
-		return false;
-
-	return btf_ctx_access(off, size, type, prog, info);
-}
-
-static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
-				 const struct bpf_reg_state *reg,
-				 int off, int size)
-{
-	return 0;
-}
-
-static const struct bpf_func_proto *
-bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
-{
-	switch (func_id) {
-	case BPF_FUNC_task_storage_get:
-		return &bpf_task_storage_get_proto;
-	case BPF_FUNC_task_storage_delete:
-		return &bpf_task_storage_delete_proto;
-	default:
-		return bpf_base_func_proto(func_id);
-	}
-}
-
-const struct bpf_verifier_ops bpf_scx_verifier_ops = {
-	.get_func_proto = bpf_scx_get_func_proto,
-	.is_valid_access = bpf_scx_is_valid_access,
-	.btf_struct_access = bpf_scx_btf_struct_access,
-};
-
-static int bpf_scx_init_member(const struct btf_type *t,
-			       const struct btf_member *member,
-			       void *kdata, const void *udata)
-{
-	const struct sched_ext_ops *uops = udata;
-	struct sched_ext_ops *ops = kdata;
-	u32 moff = __btf_member_bit_offset(t, member) / 8;
-	int ret;
-
-	switch (moff) {
-	case offsetof(struct sched_ext_ops, dispatch_max_batch):
-		if (*(u32 *)(udata + moff) > INT_MAX)
-			return -E2BIG;
-		ops->dispatch_max_batch = *(u32 *)(udata + moff);
-		return 1;
-	case offsetof(struct sched_ext_ops, flags):
-		if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS)
-			return -EINVAL;
-		ops->flags = *(u64 *)(udata + moff);
-		return 1;
-	case offsetof(struct sched_ext_ops, name):
-		ret = bpf_obj_name_cpy(ops->name, uops->name,
-				       sizeof(ops->name));
-		if (ret < 0)
-			return ret;
-		if (ret == 0)
-			return -EINVAL;
-		return 1;
-	case offsetof(struct sched_ext_ops, timeout_ms):
-		if (*(u32 *)(udata + moff) > SCX_WATCHDOG_MAX_TIMEOUT)
-			return -E2BIG;
-		ops->timeout_ms = *(u32 *)(udata + moff);
-		return 1;
-	}
-
-	return 0;
-}
-
-static int bpf_scx_check_member(const struct btf_type *t,
-				const struct btf_member *member,
-				const struct bpf_prog *prog)
-{
-	u32 moff = __btf_member_bit_offset(t, member) / 8;
-
-	switch (moff) {
-	case offsetof(struct sched_ext_ops, prep_enable):
-	case offsetof(struct sched_ext_ops, init):
-	case offsetof(struct sched_ext_ops, exit):
-		break;
-	default:
-		/* check prog->aux->sleepable int 6.2, but no prog param in 6.1 */
-		/* if (prog->aux->sleepable)
-			return -EINVAL; */
-		break;
-	}
-
-	return 0;
-}
-
-static int bpf_scx_reg(void *kdata)
-{
-	return scx_ops_enable(kdata);
-}
-
-static void bpf_scx_unreg(void *kdata)
-{
-	scx_ops_disable(SCX_EXIT_UNREG);
-	kthread_flush_work(&scx_ops_disable_work);
-}
-
-static int bpf_scx_init(struct btf *btf)
-{
-	u32 type_id;
-
-	type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT);
-	if (type_id < 0)
-		return -EINVAL;
-	task_struct_type = btf_type_by_id(btf, type_id);
-
-	return 0;
-}
-
-/* "extern" to avoid sparse warning, only used in this file */
-extern struct bpf_struct_ops bpf_sched_ext_ops;
-
-struct bpf_struct_ops bpf_sched_ext_ops = {
-	.verifier_ops = &bpf_scx_verifier_ops,
-	.reg = bpf_scx_reg,
-	.unreg = bpf_scx_unreg,
-	.check_member = bpf_scx_check_member,
-	.init_member = bpf_scx_init_member,
-	.init = bpf_scx_init,
-	.name = "sched_ext_ops",
-};
-
-static void sysrq_handle_sched_ext_reset(u8 key)
-{
-	if (scx_ops_helper)
-		scx_ops_disable(SCX_EXIT_SYSRQ);
-	else
-		pr_info("sched_ext: BPF scheduler not yet used\n");
-}
-
-static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
-	.handler	= sysrq_handle_sched_ext_reset,
-	.help_msg	= "reset-sched-ext(S)",
-	.action_msg	= "Disable sched_ext and revert all tasks to CFS",
-	.enable_mask	= SYSRQ_ENABLE_RTNICE,
-};
-
-static void kick_cpus_irq_workfn(struct irq_work *irq_work)
-{
-	struct rq *this_rq = this_rq();
-	u64 *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs);
-	int this_cpu = cpu_of(this_rq);
-	int cpu;
-
-	for_each_cpu(cpu, this_rq->scx->cpus_to_kick) {
-		struct rq *rq = cpu_rq(cpu);
-		unsigned long flags;
-
-		raw_spin_rq_lock_irqsave(rq, flags);
-
-		if (cpu_online(cpu) || cpu == this_cpu) {
-			if (cpumask_test_cpu(cpu, this_rq->scx->cpus_to_preempt) &&
-			    rq->curr->sched_class == &ext_sched_class)
-				rq->curr->scx->slice = 0;
-			pseqs[cpu] = rq->scx->pnt_seq;
-			resched_curr(rq);
-		} else {
-			cpumask_clear_cpu(cpu, this_rq->scx->cpus_to_wait);
-		}
-
-		raw_spin_rq_unlock_irqrestore(rq, flags);
-	}
-
-	for_each_cpu_andnot(cpu, this_rq->scx->cpus_to_wait,
-			    cpumask_of(this_cpu)) {
-		/*
-		 * Pairs with smp_store_release() issued by this CPU in
-		 * scx_notify_pick_next_task() on the resched path.
-		 *
-		 * We busy-wait here to guarantee that no other task can be
-		 * scheduled on our core before the target CPU has entered the
-		 * resched path.
-		 */
-		while (smp_load_acquire(&cpu_rq(cpu)->scx->pnt_seq) == pseqs[cpu])
-			cpu_relax();
-	}
-
-	cpumask_clear(this_rq->scx->cpus_to_kick);
-	cpumask_clear(this_rq->scx->cpus_to_preempt);
-	cpumask_clear(this_rq->scx->cpus_to_wait);
-}
-
-void __init init_sched_ext_class(void)
-{
-	int cpu;
-	u32 v;
-
-	/*
-	 * The following is to prevent the compiler from optimizing out the enum
-	 * definitions so that BPF scheduler implementations can use them
-	 * through the generated vmlinux.h.
-	 */
-	WRITE_ONCE(v, SCX_WAKE_EXEC | SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP |
-		   SCX_TG_ONLINE | SCX_KICK_PREEMPT);
-
-	init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL);
-#ifdef CONFIG_SMP
-	BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL));
-	BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL));
-#endif
-	scx_kick_cpus_pnt_seqs =
-		__alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) *
-			       num_possible_cpus(),
-			       __alignof__(scx_kick_cpus_pnt_seqs[0]));
-	BUG_ON(!scx_kick_cpus_pnt_seqs);
-
-	for_each_possible_cpu(cpu) {
-		struct rq *rq = cpu_rq(cpu);
-
-		rq->scx = kmalloc(sizeof(struct scx_rq), GFP_KERNEL);
-		if (rq->scx)
-			rq->scx->rq = rq;
-		else
-			pr_err("fatal error : alloc rq->scx failed!!!\n");
-
-		init_dsq(&rq->scx->local_dsq, SCX_DSQ_LOCAL);
-		INIT_LIST_HEAD(&rq->scx->watchdog_list);
-
-		BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_kick, GFP_KERNEL));
-		BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_preempt, GFP_KERNEL));
-		BUG_ON(!zalloc_cpumask_var(&rq->scx->cpus_to_wait, GFP_KERNEL));
-		init_irq_work(&rq->scx->kick_cpus_irq_work, kick_cpus_irq_workfn);
-	}
-
-	register_sysrq_key('S', &sysrq_sched_ext_reset_op);
-	INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
-	scx_cgroup_config_knobs();
-}
-
-
-/********************************************************************************
- * Helpers that can be called from the BPF scheduler.
- */
-#include <linux/btf_ids.h>
-
-/* Disables missing prototype warnings for kfuncs */
-__diag_push();
-__diag_ignore_all("-Wmissing-prototypes",
-		  "Global functions as their definitions will be in vmlinux BTF");
-
-/**
- * scx_bpf_switch_all - Switch all tasks into SCX
- * @into_scx: switch direction
- *
- * If @into_scx is %true, all existing and future non-dl/rt tasks are switched
- * to SCX. If %false, only tasks which have %SCHED_EXT explicitly set are put on
- * SCX. The actual switching is asynchronous. Can be called from ops.init().
- */
-void scx_bpf_switch_all(void)
-{
-	if (!scx_kf_allowed(SCX_KF_INIT))
-		return;
-
-	scx_switch_all_req = true;
-}
-
-BTF_SET8_START(scx_kfunc_ids_init)
-BTF_ID_FLAGS(func, scx_bpf_switch_all)
-BTF_SET8_END(scx_kfunc_ids_init)
-
-static const struct btf_kfunc_id_set scx_kfunc_set_init = {
-	.owner			= THIS_MODULE,
-	.set			= &scx_kfunc_ids_init,
-};
-
-/**
- * scx_bpf_create_dsq - Create a custom DSQ
- * @dsq_id: DSQ to create
- * @node: NUMA node to allocate from
- *
- * Create a custom DSQ identified by @dsq_id. Can be called from ops.init(),
- * ops.prep_enable(), ops.cgroup_init() and ops.cgroup_prep_move().
- */
-s32 scx_bpf_create_dsq(u64 dsq_id, s32 node)
-{
-	if (!scx_kf_allowed(SCX_KF_INIT))
-		return -EINVAL;
-
-	if (unlikely(node >= (int)nr_node_ids ||
-		     (node < 0 && node != NUMA_NO_NODE)))
-		return -EINVAL;
-	return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node));
-}
-
-BTF_SET8_START(scx_kfunc_ids_sleepable)
-BTF_ID_FLAGS(func, scx_bpf_create_dsq)
-BTF_SET8_END(scx_kfunc_ids_sleepable)
-
-static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = {
-	.owner			= THIS_MODULE,
-	.set			= &scx_kfunc_ids_sleepable,
-};
-
-static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags)
-{
-	if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH))
-		return false;
-
-	lockdep_assert_irqs_disabled();
-
-	if (unlikely(!p)) {
-		scx_ops_error("called with NULL task");
-		return false;
-	}
-
-	if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
-		scx_ops_error("invalid enq_flags 0x%llx", enq_flags);
-		return false;
-	}
-
-	return true;
-}
-
-static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags)
-{
-	struct task_struct *ddsp_task;
-	int idx;
-
-	ddsp_task = __this_cpu_read(direct_dispatch_task);
-	if (ddsp_task) {
-		direct_dispatch(ddsp_task, p, dsq_id, enq_flags);
-		return;
-	}
-
-	idx = __this_cpu_read(scx_dsp_ctx.buf_cursor);
-	if (unlikely(idx >= scx_dsp_max_batch)) {
-		scx_ops_error("dispatch buffer overflow");
-		return;
-	}
-
-	this_cpu_ptr(scx_dsp_buf)[idx] = (struct scx_dsp_buf_ent){
-		.task = p,
-		.qseq = atomic64_read(&p->scx->ops_state) & SCX_OPSS_QSEQ_MASK,
-		.dsq_id = dsq_id,
-		.enq_flags = enq_flags,
-	};
-	__this_cpu_inc(scx_dsp_ctx.buf_cursor);
-}
-
-/**
- * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ
- * @p: task_struct to dispatch
- * @dsq_id: DSQ to dispatch to
- * @slice: duration @p can run for in nsecs
- * @enq_flags: SCX_ENQ_*
- *
- * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe
- * to call this function spuriously. Can be called from ops.enqueue() and
- * ops.dispatch().
- *
- * When called from ops.enqueue(), it's for direct dispatch and @p must match
- * the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be used to target the
- * local DSQ of a CPU other than the enqueueing one. Use ops.select_cpu() to be
- * on the target CPU in the first place.
- *
- * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
- * and this function can be called upto ops.dispatch_max_batch times to dispatch
- * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
- * remaining slots. scx_bpf_consume() flushes the batch and resets the counter.
- *
- * This function doesn't have any locking restrictions and may be called under
- * BPF locks (in the future when BPF introduces more flexible locking).
- *
- * @p is allowed to run for @slice. The scheduling path is triggered on slice
- * exhaustion. If zero, the current residual slice is maintained. If
- * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with
- * scx_bpf_kick_cpu() to trigger scheduling.
- */
-void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice,
-		      u64 enq_flags)
-{
-	if (!scx_dispatch_preamble(p, enq_flags))
-		return;
-
-	if (slice)
-		p->scx->slice = slice;
-	else
-		p->scx->slice = p->scx->slice ?: 1;
-
-	scx_dispatch_commit(p, dsq_id, enq_flags);
-}
-
-/**
- * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ
- * @p: task_struct to dispatch
- * @dsq_id: DSQ to dispatch to
- * @slice: duration @p can run for in nsecs
- * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
- * @enq_flags: SCX_ENQ_*
- *
- * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id.
- * Tasks queued into the priority queue are ordered by @vtime and always
- * consumed after the tasks in the FIFO queue. All other aspects are identical
- * to scx_bpf_dispatch().
- *
- * @vtime ordering is according to time_before64() which considers wrapping. A
- * numerically larger vtime may indicate an earlier position in the ordering and
- * vice-versa.
- */
-void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice,
-			    u64 vtime, u64 enq_flags)
-{
-	if (!scx_dispatch_preamble(p, enq_flags))
-		return;
-
-	if (slice)
-		p->scx->slice = slice;
-	else
-		p->scx->slice = p->scx->slice ?: 1;
-
-	p->scx->dsq_vtime = vtime;
-
-	scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
-}
-
-BTF_SET8_START(scx_kfunc_ids_enqueue_dispatch)
-BTF_ID_FLAGS(func, scx_bpf_dispatch)
-BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime)
-BTF_SET8_END(scx_kfunc_ids_enqueue_dispatch)
-
-static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
-	.owner			= THIS_MODULE,
-	.set			= &scx_kfunc_ids_enqueue_dispatch,
-};
-
-/**
- * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots
- *
- * Can only be called from ops.dispatch().
- */
-u32 scx_bpf_dispatch_nr_slots(void)
-{
-	if (!scx_kf_allowed(SCX_KF_DISPATCH))
-		return 0;
-
-	return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx.buf_cursor);
-}
-
-/**
- * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ
- * @dsq_id: DSQ to consume
- *
- * Consume a task from the non-local DSQ identified by @dsq_id and transfer it
- * to the current CPU's local DSQ for execution. Can only be called from
- * ops.dispatch().
- *
- * This function flushes the in-flight dispatches from scx_bpf_dispatch() before
- * trying to consume the specified DSQ. It may also grab rq locks and thus can't
- * be called under any BPF locks.
- *
- * Returns %true if a task has been consumed, %false if there isn't any task to
- * consume.
- */
-bool scx_bpf_consume(u64 dsq_id)
-{
-	struct scx_dsp_ctx *dspc = this_cpu_ptr(&scx_dsp_ctx);
-	struct scx_dispatch_q *dsq;
-
-	if (!scx_kf_allowed(SCX_KF_DISPATCH))
-		return false;
-
-	flush_dispatch_buf(dspc->rq, dspc->rf);
-
-	dsq = find_non_local_dsq(dsq_id);
-	if (unlikely(!dsq)) {
-		scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id);
-		return false;
-	}
-
-	if (consume_dispatch_q(dspc->rq, dspc->rf, dsq)) {
-		/*
-		 * A successfully consumed task can be dequeued before it starts
-		 * running while the CPU is trying to migrate other dispatched
-		 * tasks. Bump nr_tasks to tell balance_scx() to retry on empty
-		 * local DSQ.
-		 */
-		dspc->nr_tasks++;
-		return true;
-	} else {
-		return false;
-	}
-}
-
-BTF_SET8_START(scx_kfunc_ids_dispatch)
-BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots)
-BTF_ID_FLAGS(func, scx_bpf_consume)
-BTF_SET8_END(scx_kfunc_ids_dispatch)
-
-static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
-	.owner			= THIS_MODULE,
-	.set			= &scx_kfunc_ids_dispatch,
-};
-
-/**
- * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
- *
- * Iterate over all of the tasks currently enqueued on the local DSQ of the
- * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of
- * processed tasks. Can only be called from ops.cpu_release().
- */
-u32 scx_bpf_reenqueue_local(void)
-{
-	u32 nr_enqueued, i;
-	struct rq *rq;
-	struct scx_rq *scx_rq;
-
-	if (!scx_kf_allowed(SCX_KF_CPU_RELEASE))
-		return 0;
-
-	rq = cpu_rq(smp_processor_id());
-	lockdep_assert_rq_held(rq);
-	scx_rq = rq->scx;
-
-	/*
-	 * Get the number of tasks on the local DSQ before iterating over it to
-	 * pull off tasks. The enqueue callback below can signal that it wants
-	 * the task to stay on the local DSQ, and we want to prevent the BPF
-	 * scheduler from causing us to loop indefinitely.
-	 */
-	nr_enqueued = scx_rq->local_dsq.nr;
-	for (i = 0; i < nr_enqueued; i++) {
-		struct task_struct *p;
-
-		p = first_local_task(rq);
-		WARN_ON_ONCE(atomic64_read(&p->scx->ops_state) != SCX_OPSS_NONE);
-		WARN_ON_ONCE(!(p->scx->flags & SCX_TASK_QUEUED));
-		WARN_ON_ONCE(p->scx->holding_cpu != -1);
-		dispatch_dequeue(scx_rq, p);
-		do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
-	}
-
-	return nr_enqueued;
-}
-
-BTF_SET8_START(scx_kfunc_ids_cpu_release)
-BTF_ID_FLAGS(func, scx_bpf_reenqueue_local)
-BTF_SET8_END(scx_kfunc_ids_cpu_release)
-
-static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = {
-	.owner			= THIS_MODULE,
-	.set			= &scx_kfunc_ids_cpu_release,
-};
-
-/**
- * scx_bpf_kick_cpu - Trigger reschedule on a CPU
- * @cpu: cpu to kick
- * @flags: SCX_KICK_* flags
- *
- * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
- * trigger rescheduling on a busy CPU. This can be called from any online
- * scx_ops operation and the actual kicking is performed asynchronously through
- * an irq work.
- */
-void scx_bpf_kick_cpu(s32 cpu, u64 flags)
-{
-	struct rq *rq;
-
-	if (!ops_cpu_valid(cpu)) {
-		scx_ops_error("invalid cpu %d", cpu);
-		return;
-	}
-
-	preempt_disable();
-	rq = this_rq();
-
-	/*
-	 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting
-	 * rq locks. We can probably be smarter and avoid bouncing if called
-	 * from ops which don't hold a rq lock.
-	 */
-	cpumask_set_cpu(cpu, rq->scx->cpus_to_kick);
-	if (flags & SCX_KICK_PREEMPT)
-		cpumask_set_cpu(cpu, rq->scx->cpus_to_preempt);
-	if (flags & SCX_KICK_WAIT)
-		cpumask_set_cpu(cpu, rq->scx->cpus_to_wait);
-
-	irq_work_queue(&rq->scx->kick_cpus_irq_work);
-	preempt_enable();
-}
-
-/**
- * scx_bpf_dsq_nr_queued - Return the number of queued tasks
- * @dsq_id: id of the DSQ
- *
- * Return the number of tasks in the DSQ matching @dsq_id. If not found,
- * -%ENOENT is returned. Can be called from any non-sleepable online scx_ops
- * operations.
- */
-s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
-{
-	struct scx_dispatch_q *dsq;
-
-	lockdep_assert(rcu_read_lock_any_held());
-
-	if (dsq_id == SCX_DSQ_LOCAL) {
-		return this_rq()->scx->local_dsq.nr;
-	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
-		s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
-
-		if (ops_cpu_valid(cpu))
-			return cpu_rq(cpu)->scx->local_dsq.nr;
-	} else {
-		dsq = find_non_local_dsq(dsq_id);
-		if (dsq)
-			return dsq->nr;
-	}
-	return -ENOENT;
-}
-
-/**
- * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state
- * @cpu: cpu to test and clear idle for
- *
- * Returns %true if @cpu was idle and its idle state was successfully cleared.
- * %false otherwise.
- *
- * Unavailable if ops.update_idle() is implemented and
- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
- */
-bool scx_bpf_test_and_clear_cpu_idle(s32 cpu)
-{
-	if (!static_branch_likely(&scx_builtin_idle_enabled)) {
-		scx_ops_error("built-in idle tracking is disabled");
-		return false;
-	}
-
-	if (ops_cpu_valid(cpu))
-		return test_and_clear_cpu_idle(cpu);
-	else
-		return false;
-}
-
-/**
- * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu
- * @cpus_allowed: Allowed cpumask
- *
- * Pick and claim an idle cpu which is also in @cpus_allowed. Returns the picked
- * idle cpu number on success. -%EBUSY if no matching cpu was found.
- *
- * Unavailable if ops.update_idle() is implemented and
- * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
- */
-s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed)
-{
-	if (!static_branch_likely(&scx_builtin_idle_enabled)) {
-		scx_ops_error("built-in idle tracking is disabled");
-		return -EBUSY;
-	}
-
-	return scx_pick_idle_cpu(cpus_allowed);
-}
-
-/**
- * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking
- * per-CPU cpumask.
- *
- * Returns NULL if idle tracking is not enabled, or running on a UP kernel.
- */
-const struct cpumask *scx_bpf_get_idle_cpumask(void)
-{
-	if (!static_branch_likely(&scx_builtin_idle_enabled)) {
-		scx_ops_error("built-in idle tracking is disabled");
-		return cpu_none_mask;
-	}
-
-#ifdef CONFIG_SMP
-	return idle_masks.cpu;
-#else
-	return cpu_none_mask;
-#endif
-}
-
-/**
- * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking,
- * per-physical-core cpumask. Can be used to determine if an entire physical
- * core is free.
- *
- * Returns NULL if idle tracking is not enabled, or running on a UP kernel.
- */
-const struct cpumask *scx_bpf_get_idle_smtmask(void)
-{
-	if (!static_branch_likely(&scx_builtin_idle_enabled)) {
-		scx_ops_error("built-in idle tracking is disabled");
-		return cpu_none_mask;
-	}
-
-#ifdef CONFIG_SMP
-	return idle_masks.smt;
-#else
-	return cpu_none_mask;
-#endif
-}
-
-/**
- * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to
- * either the percpu, or SMT idle-tracking cpumask.
- */
-void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask)
-{
-	/*
-	 * Empty function body because we aren't actually acquiring or
-	 * releasing a reference to a global idle cpumask, which is read-only
-	 * in the caller and is never released. The acquire / release semantics
-	 * here are just used to make the cpumask is a trusted pointer in the
-	 * caller.
-	 */
-}
-
-struct scx_bpf_error_bstr_bufs {
-	u64			data[MAX_BPRINTF_VARARGS];
-	char			msg[SCX_EXIT_MSG_LEN];
-};
-
-static DEFINE_PER_CPU(struct scx_bpf_error_bstr_bufs, scx_bpf_error_bstr_bufs);
-
-/**
- * scx_bpf_error_bstr - Indicate fatal error
- * @fmt: error message format string
- * @data: format string parameters packaged using ___bpf_fill() macro
- * @data__sz: @data len, must end in '__sz' for the verifier
- *
- * Indicate that the BPF scheduler encountered a fatal error and initiate ops
- * disabling.
- */
-void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data__sz)
-{
-	struct scx_bpf_error_bstr_bufs *bufs;
-	unsigned long flags;
-	struct bpf_bprintf_data args = {};
-	int ret;
-
-	local_irq_save(flags);
-	bufs = this_cpu_ptr(&scx_bpf_error_bstr_bufs);
-
-	if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 ||
-	    (data__sz && !data)) {
-		scx_ops_error("invalid data=%p and data__sz=%u",
-			      (void *)data, data__sz);
-		goto out_restore;
-	}
-
-	ret = copy_from_kernel_nofault(bufs->data, data, data__sz);
-	if (ret) {
-		scx_ops_error("failed to read data fields (%d)", ret);
-		goto out_restore;
-	}
-
-	ret = bpf_bprintf_prepare(fmt, UINT_MAX, bufs->data, data__sz / 8, &args);
-	if (ret < 0) {
-		scx_ops_error("failed to format prepration (%d)", ret);
-		goto out_restore;
-	}
-
-	ret = bstr_printf(bufs->msg, sizeof(bufs->msg), fmt,
-			  args.bin_args);
-	bpf_bprintf_cleanup(&args);
-	if (ret < 0) {
-		scx_ops_error("scx_ops_error(\"%s\", %p, %u) failed to format",
-			      fmt, data, data__sz);
-		goto out_restore;
-	}
-
-	scx_ops_error_type(SCX_EXIT_ERROR_BPF, "%s", bufs->msg);
-out_restore:
-	local_irq_restore(flags);
-}
-
-/**
- * scx_bpf_destroy_dsq - Destroy a custom DSQ
- * @dsq_id: DSQ to destroy
- *
- * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
- * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
- * empty and no further tasks are dispatched to it. Ignored if called on a DSQ
- * which doesn't exist. Can be called from any online scx_ops operations.
- */
-void scx_bpf_destroy_dsq(u64 dsq_id)
-{
-	destroy_dsq(dsq_id);
-}
-
-/**
- * scx_bpf_task_running - Is task currently running?
- * @p: task of interest
- */
-bool scx_bpf_task_running(const struct task_struct *p)
-{
-	return task_rq(p)->curr == p;
-}
-
-/**
- * scx_bpf_task_cpu - CPU a task is currently associated with
- * @p: task of interest
- */
-s32 scx_bpf_task_cpu(const struct task_struct *p)
-{
-	return task_cpu(p);
-}
-
-/**
- * scx_bpf_task_cgroup - Return the sched cgroup of a task
- * @p: task of interest
- *
- * @p->sched_task_group->css.cgroup represents the cgroup @p is associated with
- * from the scheduler's POV. SCX operations should use this function to
- * determine @p's current cgroup as, unlike following @p->cgroups,
- * @p->sched_task_group is protected by @p's rq lock and thus atomic w.r.t. all
- * rq-locked operations. Can be called on the parameter tasks of rq-locked
- * operations. The restriction guarantees that @p's rq is locked by the caller.
- */
-struct cgroup *scx_bpf_task_cgroup(struct task_struct *p)
-{
-	struct task_group *tg = p->sched_task_group;
-	struct cgroup *cgrp = &cgrp_dfl_root.cgrp;
-
-	if (!scx_kf_allowed_on_arg_tasks(__SCX_KF_RQ_LOCKED, p))
-		goto out;
-
-	/*
-	 * A task_group may either be a cgroup or an autogroup. In the latter
-	 * case, @tg->css.cgroup is %NULL. A task_group can't become the other
-	 * kind once created.
-	 */
-	if (tg && tg->css.cgroup)
-		cgrp = tg->css.cgroup;
-	else
-		cgrp = &cgrp_dfl_root.cgrp;
-out:
-	cgroup_get(cgrp);
-	return cgrp;
-}
-
-BTF_SET8_START(scx_kfunc_ids_any)
-BTF_ID_FLAGS(func, scx_bpf_kick_cpu)
-BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
-BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle)
-BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu)
-BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE)
-BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE)
-BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
-BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS)
-BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
-BTF_ID_FLAGS(func, scx_bpf_task_running)
-BTF_ID_FLAGS(func, scx_bpf_task_cpu)
-BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_ACQUIRE)
-BTF_SET8_END(scx_kfunc_ids_any)
-
-static const struct btf_kfunc_id_set scx_kfunc_set_any = {
-	.owner			= THIS_MODULE,
-	.set			= &scx_kfunc_ids_any,
-};
-
-__diag_pop();
-
-/*
- * This can't be done from init_sched_ext_class() as register_btf_kfunc_id_set()
- * needs most of the system to be up.
- */
-static int __init register_ext_kfuncs(void)
-{
-	int ret;
-
-	/*
-	 * Some kfuncs are context-sensitive and can only be called from
-	 * specific SCX ops. They are grouped into BTF sets accordingly.
-	 * Unfortunately, BPF currently doesn't have a way of enforcing such
-	 * restrictions. Eventually, the verifier should be able to enforce
-	 * them. For now, register them the same and make each kfunc explicitly
-	 * check using scx_kf_allowed().
-	 */
-	if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					     &scx_kfunc_set_init)) ||
-	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					     &scx_kfunc_set_sleepable)) ||
-	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					     &scx_kfunc_set_enqueue_dispatch)) ||
-	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					     &scx_kfunc_set_dispatch)) ||
-	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					     &scx_kfunc_set_cpu_release)) ||
-	    (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
-					     &scx_kfunc_set_any))) {
-		pr_err("sched_ext: failed to register kfunc sets (%d)\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-__initcall(register_ext_kfuncs);
diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h
deleted file mode 100755
index fba8ed845f99..000000000000
--- a/kernel/sched/ext.h
+++ /dev/null
@@ -1,257 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-
-
-/*
- * Tag marking a kernel function as a kfunc. This is meant to minimize the
- * amount of copy-paste that kfunc authors have to include for correctness so
- * as to avoid issues such as the compiler inlining or eliding either a static
- * kfunc, or a global kfunc in an LTO build.
- */
-#define __bpf_kfunc __used noinline
-
-enum scx_wake_flags {
-	/* expose select WF_* flags as enums */
-	SCX_WAKE_EXEC		= WF_EXEC,
-	SCX_WAKE_FORK		= WF_FORK,
-	SCX_WAKE_TTWU		= WF_TTWU,
-	SCX_WAKE_SYNC		= WF_SYNC,
-};
-
-enum scx_enq_flags {
-	/* expose select ENQUEUE_* flags as enums */
-	SCX_ENQ_WAKEUP		= ENQUEUE_WAKEUP,
-	SCX_ENQ_HEAD		= ENQUEUE_HEAD,
-
-	/* high 32bits are SCX specific */
-
-	/*
-	 * Set the following to trigger preemption when calling
-	 * scx_bpf_dispatch() with a local dsq as the target. The slice of the
-	 * current task is cleared to zero and the CPU is kicked into the
-	 * scheduling path. Implies %SCX_ENQ_HEAD.
-	 */
-	SCX_ENQ_PREEMPT		= 1LLU << 32,
-
-	/*
-	 * The task being enqueued was previously enqueued on the current CPU's
-	 * %SCX_DSQ_LOCAL, but was removed from it in a call to the
-	 * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was
-	 * invoked in a ->cpu_release() callback, and the task is again
-	 * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
-	 * task will not be scheduled on the CPU until at least the next invocation
-	 * of the ->cpu_acquire() callback.
-	 */
-	SCX_ENQ_REENQ		= 1LLU << 40,
-
-	/*
-	 * The task being enqueued is the only task available for the cpu. By
-	 * default, ext core keeps executing such tasks but when
-	 * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with
-	 * %SCX_ENQ_LAST and %SCX_ENQ_LOCAL flags set.
-	 *
-	 * If the BPF scheduler wants to continue executing the task,
-	 * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately.
-	 * If the task gets queued on a different dsq or the BPF side, the BPF
-	 * scheduler is responsible for triggering a follow-up scheduling event.
-	 * Otherwise, Execution may stall.
-	 */
-	SCX_ENQ_LAST		= 1LLU << 41,
-
-	/*
-	 * A hint indicating that it's advisable to enqueue the task on the
-	 * local dsq of the currently selected CPU. Currently used by
-	 * select_cpu_dfl() and together with %SCX_ENQ_LAST.
-	 */
-	SCX_ENQ_LOCAL		= 1LLU << 42,
-
-	/* high 8 bits are internal */
-	__SCX_ENQ_INTERNAL_MASK	= 0xffLLU << 56,
-
-	SCX_ENQ_CLEAR_OPSS	= 1LLU << 56,
-	SCX_ENQ_DSQ_PRIQ	= 1LLU << 57,
-};
-
-enum scx_deq_flags {
-	/* expose select DEQUEUE_* flags as enums */
-	SCX_DEQ_SLEEP		= DEQUEUE_SLEEP,
-
-	/* high 32bits are SCX specific */
-
-	/*
-	 * The generic core-sched layer decided to execute the task even though
-	 * it hasn't been dispatched yet. Dequeue from the BPF side.
-	 */
-	SCX_DEQ_CORE_SCHED_EXEC	= 1LLU << 32,
-};
-
-enum scx_tg_flags {
-	SCX_TG_ONLINE		= 1U << 0,
-	SCX_TG_INITED		= 1U << 1,
-};
-
-enum scx_kick_flags {
-	SCX_KICK_PREEMPT	= 1LLU << 0,	/* force scheduling on the CPU */
-	SCX_KICK_WAIT		= 1LLU << 1,	/* wait for the CPU to be rescheduled */
-};
-
-#ifdef CONFIG_SCHED_CLASS_EXT
-
-extern const struct sched_class ext_sched_class;
-extern const struct bpf_verifier_ops bpf_sched_ext_verifier_ops;
-extern const struct file_operations sched_ext_fops;
-extern unsigned long scx_watchdog_timeout;
-extern unsigned long scx_watchdog_timestamp;
-
-DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled);
-DECLARE_STATIC_KEY_FALSE(__scx_switched_all);
-#define scx_enabled()		static_branch_unlikely(&__scx_ops_enabled)
-#define scx_switched_all()	static_branch_unlikely(&__scx_switched_all)
-
-DECLARE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);
-
-bool task_on_scx(struct task_struct *p);
-void scx_pre_fork(struct task_struct *p);
-int scx_fork(struct task_struct *p);
-void scx_post_fork(struct task_struct *p);
-void scx_cancel_fork(struct task_struct *p);
-int scx_check_setscheduler(struct task_struct *p, int policy);
-bool scx_can_stop_tick(struct rq *rq);
-void init_sched_ext_class(void);
-
-__printf(2, 3) void scx_ops_error_type(enum scx_exit_type type,
-				       const char *fmt, ...);
-#define scx_ops_error(fmt, args...)						\
-	scx_ops_error_type(SCX_EXIT_ERROR, fmt, ##args)
-
-void __scx_notify_pick_next_task(struct rq *rq,
-				 struct task_struct *p,
-				 const struct sched_class *active);
-
-static inline void scx_notify_pick_next_task(struct rq *rq,
-					     struct task_struct *p,
-					     const struct sched_class *active)
-{
-	if (!scx_enabled())
-		return;
-#ifdef CONFIG_SMP
-	/*
-	 * Pairs with the smp_load_acquire() issued by a CPU in
-	 * kick_cpus_irq_workfn() who is waiting for this CPU to perform a
-	 * resched.
-	 */
-	smp_store_release(&rq->scx->pnt_seq, rq->scx->pnt_seq + 1);
-#endif
-	if (!static_branch_unlikely(&scx_ops_cpu_preempt))
-		return;
-	__scx_notify_pick_next_task(rq, p, active);
-}
-
-//extern void scx_scheduler_tick(void);
-static inline void scx_notify_sched_tick(void)
-{
-	unsigned long last_check;
-
-	if (!scx_enabled())
-		return;
-	//scx_scheduler_tick();
-
-	last_check = scx_watchdog_timestamp;
-	if (unlikely(time_after(jiffies, last_check + scx_watchdog_timeout))) {
-		u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
-
-		scx_ops_error_type(SCX_EXIT_ERROR_STALL,
-				   "watchdog failed to check in for %u.%03us",
-				   dur_ms / 1000, dur_ms % 1000);
-	}
-}
-
-static inline const struct sched_class *next_active_class(const struct sched_class *class)
-{
-	class++;
-	if (scx_switched_all() && class == &fair_sched_class)
-		class++;
-	if (!scx_enabled() && class == &ext_sched_class)
-		class++;
-	return class;
-}
-
-#define for_active_class_range(class, _from, _to)				\
-	for (class = (_from); class != (_to); class = next_active_class(class))
-
-#define for_each_active_class(class)						\
-	for_active_class_range(class, __sched_class_highest, __sched_class_lowest)
-
-/*
- * SCX requires a balance() call before every pick_next_task() call including
- * when waking up from idle.
- */
-#define for_balance_class_range(class, prev_class, end_class)			\
-	for_active_class_range(class, (prev_class) > &ext_sched_class ?		\
-			       &ext_sched_class : (prev_class), (end_class))
-
-#ifdef CONFIG_SCHED_CORE
-bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
-		   bool in_fi);
-#endif
-
-#else	/* CONFIG_SCHED_CLASS_EXT */
-
-#define scx_enabled()		false
-#define scx_switched_all()	false
-
-static inline void scx_pre_fork(struct task_struct *p) {}
-static inline int scx_fork(struct task_struct *p) { return 0; }
-static inline void scx_post_fork(struct task_struct *p) {}
-static inline void scx_cancel_fork(struct task_struct *p) {}
-static inline int scx_check_setscheduler(struct task_struct *p,
-					 int policy) { return 0; }
-static inline bool scx_can_stop_tick(struct rq *rq) { return true; }
-static inline void init_sched_ext_class(void) {}
-static inline void scx_notify_pick_next_task(struct rq *rq,
-					     const struct task_struct *p,
-					     const struct sched_class *active) {}
-static inline void scx_notify_sched_tick(void) {}
-
-#define for_each_active_class		for_each_class
-#define for_balance_class_range		for_class_range
-
-#endif	/* CONFIG_SCHED_CLASS_EXT */
-
-#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP)
-void __scx_update_idle(struct rq *rq, bool idle);
-
-static inline void scx_update_idle(struct rq *rq, bool idle)
-{
-	if (scx_enabled())
-		__scx_update_idle(rq, idle);
-}
-#else
-static inline void scx_update_idle(struct rq *rq, bool idle) {}
-#endif
-
-#ifdef CONFIG_CGROUP_SCHED
-#ifdef CONFIG_EXT_GROUP_SCHED
-int scx_tg_online(struct task_group *tg);
-void scx_tg_offline(struct task_group *tg);
-int scx_cgroup_can_attach(struct cgroup_taskset *tset);
-void scx_move_task(struct task_struct *p);
-void scx_cgroup_finish_attach(void);
-void scx_cgroup_cancel_attach(struct cgroup_taskset *tset);
-void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight);
-#else	/* CONFIG_EXT_GROUP_SCHED */
-static inline int scx_tg_online(struct task_group *tg) { return 0; }
-static inline void scx_tg_offline(struct task_group *tg) {}
-static inline int scx_cgroup_can_attach(struct cgroup_taskset *tset) { return 0; }
-static inline void scx_move_task(struct task_struct *p) {}
-static inline void scx_cgroup_finish_attach(void) {}
-static inline void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) {}
-static inline void scx_group_set_weight(struct task_group *tg, unsigned long cgrp_weight) {}
-#endif	/* CONFIG_EXT_GROUP_SCHED */
-#endif	/* CONFIG_CGROUP_SCHED */
diff --git a/kernel/sched/hmbird.h b/kernel/sched/hmbird.h
new file mode 100755
index 000000000000..d0d84ddee13d
--- /dev/null
+++ b/kernel/sched/hmbird.h
@@ -0,0 +1,343 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * HMBIRD scheduler class
+ *
+ * Copyright (c) 2024 OPlus.
+ * Copyright (c) 2024 Dao Huang
+ * Copyright (c) 2024 Yuxing Wang
+ * Copyright (c) 2024 Taiyu Li
+ */
+
+#ifndef _HMBIRD_H_
+#define _HMBIRD_H_
+
+/*
+ * Tag marking a kernel function as a kfunc. This is meant to minimize the
+ * amount of copy-paste that kfunc authors have to include for correctness so
+ * as to avoid issues such as the compiler inlining or eliding either a static
+ * kfunc, or a global kfunc in an LTO build.
+ */
+
+#define DEBUG_INTERNAL		(1 << 0)
+#define	DEBUG_INFO_TRACE	(1 << 1)
+#define DEBUG_INFO_SYSTRACE	(1 << 2)
+
+#define NUMS_CGROUP_KINDS	(512)
+#define SLIM_FOR_SGAME		(1 << 0)
+#define SLIM_FOR_GENSHIN	(1 << 1)
+
+#ifdef MAX_TASK_NR
+#define MAX_KEY_THREAD_RECORD		((MAX_TASK_NR + 1) >> 1)
+#else
+#define MAX_KEY_THREAD_RECORD		(1 << 3)
+#endif /* MAX_TASK_NR */
+
+#define TOP_TASK_SHIFT				(8)
+#define TOP_TASK_MAX				(1 << TOP_TASK_SHIFT)
+#ifndef TOP_TASK_BITS_MASK
+#define	TOP_TASK_BITS_MASK			(TOP_TASK_MAX - 1)
+#endif /* TOP_TASK_BITS_MASK */
+
+extern struct md_info_t *md_info;
+
+#define debug_enabled()	\
+	(unlikely(hmbirdcore_debug))
+
+#define hmbird_debug(fmt, ...)	\
+	pr_info("<hmbird_sched>:"fmt, ##__VA_ARGS__)
+
+#define hmbird_err(id, fmt, ...) \
+do {							\
+	pr_err("<hmbird_sched>"fmt, ##__VA_ARGS__);	\
+	exceps_update(md_info, id, jiffies);	\
+} while (0)
+
+#define hmbird_deferred_err(id, fmt, ...) \
+do {					\
+	printk_deferred("<hmbird_sched>"fmt, ##__VA_ARGS__);	\
+	exceps_update(md_info, id, jiffies);	\
+} while (0)
+
+#define hmbird_cond_deferred_err(id, cond, fmt, ...) \
+do {							\
+	if (unlikely(cond)) {				\
+		hmbird_deferred_err(id, #cond","fmt, ##__VA_ARGS__);	\
+		exceps_update(md_info, id, jiffies);	\
+	}							\
+} while (0)
+
+#ifdef CONFIG_HMBIRD_DEBUG_MODE
+#define hmbird_info_trace(fmt, ...)			\
+do {						\
+	if (unlikely(hmbirdcore_debug & DEBUG_INFO_TRACE))		\
+		trace_printk("<hmbird_sched>:"fmt, ##__VA_ARGS__); \
+} while (0)
+#else
+#define hmbird_info_trace(fmt, ...)
+#endif
+
+#define hmbird_info_systrace(fmt, ...)	\
+do {					\
+	if (unlikely(hmbirdcore_debug & DEBUG_INFO_SYSTRACE)) {	\
+		char buf[256];		\
+		snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__);	\
+		tracing_mark_write(buf);			\
+	}				\
+} while (0)
+
+#define hmbird_output_systrace(fmt, ...)	\
+do {					\
+	char buf[256];		\
+	snprintf(buf, sizeof(buf), fmt, ##__VA_ARGS__);	\
+	tracing_mark_write(buf);			\
+} while (0)
+
+#ifdef CONFIG_HMBIRD_DEBUG_MODE
+#define hmbird_internal_trace(fmt, ...)			\
+do {						\
+	if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL))		\
+		trace_printk("<hmbird_sched>:"fmt, ##__VA_ARGS__); \
+} while (0)
+#else
+#define hmbird_internal_trace(fmt, ...)
+#endif
+
+#define hmbird_internal_systrace(fmt, ...)			\
+do {								\
+	if (unlikely(hmbirdcore_debug & DEBUG_INTERNAL)) {	\
+		hmbird_output_systrace(fmt, ##__VA_ARGS__);	\
+	}							\
+} while (0)
+
+enum hmbird_wake_flags {
+	/* expose select WF_* flags as enums */
+	HMBIRD_WAKE_EXEC		= WF_EXEC,
+	HMBIRD_WAKE_FORK		= WF_FORK,
+	HMBIRD_WAKE_TTWU		= WF_TTWU,
+	HMBIRD_WAKE_SYNC		= WF_SYNC,
+};
+
+enum hmbird_enq_flags {
+	/* expose select ENQUEUE_* flags as enums */
+	HMBIRD_ENQ_WAKEUP		= ENQUEUE_WAKEUP,
+	HMBIRD_ENQ_HEAD		= ENQUEUE_HEAD,
+
+	/* high 32bits are HMBIRD specific */
+
+	/*
+	 * Set the following to trigger preemption when calling
+	 * hmbird_bpf_dispatch() with a local dsq as the target. The slice of the
+	 * current task is cleared to zero and the CPU is kicked into the
+	 * scheduling path. Implies %HMBIRD_ENQ_HEAD.
+	 */
+	HMBIRD_ENQ_PREEMPT		= 1LLU << 32,
+	HMBIRD_ENQ_REENQ		= 1LLU << 40,
+	HMBIRD_ENQ_LAST		= 1LLU << 41,
+
+	/*
+	 * A hint indicating that it's advisable to enqueue the task on the
+	 * local dsq of the currently selected CPU. Currently used by
+	 * select_cpu_dfl() and together with %HMBIRD_ENQ_LAST.
+	 */
+	HMBIRD_ENQ_LOCAL		= 1LLU << 42,
+
+	/* high 8 bits are internal */
+	__HMBIRD_ENQ_INTERNAL_MASK	= 0xffLLU << 56,
+
+	HMBIRD_ENQ_CLEAR_OPSS	= 1LLU << 56,
+	HMBIRD_ENQ_DSQ_PRIQ	= 1LLU << 57,
+};
+
+enum hmbird_deq_flags {
+	/* expose select DEQUEUE_* flags as enums */
+	HMBIRD_DEQ_SLEEP		= DEQUEUE_SLEEP,
+
+	/* high 32bits are HMBIRD specific */
+
+	/*
+	 * The generic core-sched layer decided to execute the task even though
+	 * it hasn't been dispatched yet. Dequeue from the BPF side.
+	 */
+	HMBIRD_DEQ_CORE_SCHED_EXEC	= 1LLU << 32,
+};
+
+enum hmbird_kick_flags {
+	HMBIRD_KICK_PREEMPT	= 1LLU << 0,	/* force scheduling on the CPU */
+	HMBIRD_KICK_WAIT		= 1LLU << 1,	/* wait for the CPU to be rescheduled */
+};
+
+enum hmbird_task_prop_type {
+	HMBIRD_TASK_PROP_COMMON,
+	HMBIRD_TASK_PROP_PIPELINE,
+	HMBIRD_TASK_PROP_DEBUG_OR_LOG,
+	HMBIRD_TASK_PROP_HIGH_LOAD,
+	HMBIRD_TASK_PROP_IO,
+	HMBIRD_TASK_PROP_NETWORK,
+	HMBIRD_TASK_PROP_PERIODIC,
+	HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL,
+	HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL,
+	HMBIRD_TASK_PROP_ISOLATE,
+	HMBIRD_TASK_PROP_MAX,
+};
+
+enum hmbird_preempt_policy_id {
+	HMBIRD_PREEMPT_POLICY_NONE,
+	HMBIRD_PREEMPT_POLICY_PRIO_BASED,
+};
+
+extern const struct sched_class hmbird_sched_class;
+extern const struct bpf_verifier_ops bpf_sched_hmbird_verifier_ops;
+extern const struct file_operations sched_hmbird_fops;
+extern unsigned long hmbird_watchdog_timeout;
+extern unsigned long hmbird_watchdog_timestamp;
+
+enum scx_rq_flags {
+	HMBIRD_RQ_CAN_STOP_TICK	= 1 << 0,
+};
+
+struct hmbird_rq {
+	struct hmbird_dispatch_q	local_dsq;
+	struct list_head	watchdog_list;
+	u64			ops_qseq;
+	u64			extra_enq_flags;	/* see move_task_to_local_dsq() */
+	u32			nr_running;
+	u32			flags;
+	bool			cpu_released;
+	cpumask_var_t		cpus_to_kick;
+	cpumask_var_t		cpus_to_preempt;
+	cpumask_var_t		cpus_to_wait;
+	u64			pnt_seq;
+	u64			*prev_runnable_sum_fixed;
+	u32			*prev_window_size;
+	struct irq_work		kick_cpus_irq_work;
+	bool			pipeline;
+	bool			exclusive;
+	bool			period_disallow;
+	bool			nonperiod_disallow;
+	struct rq		*rq;
+	struct hmbird_sched_rq_stats	*srq;
+};
+
+struct hmbird_sched_change_guard {
+	struct task_struct		*p;
+	struct rq				*rq;
+	bool					queued;
+	bool					running;
+	bool					done;
+};
+
+extern struct hmbird_sched_change_guard
+hmbird_sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags);
+
+extern void hmbird_sched_change_guard_fini(struct hmbird_sched_change_guard *cg, int flags);
+
+#define SCHED_CHANGE_BLOCK(__rq, __p, __flags)				\
+	for (struct hmbird_sched_change_guard __cg =			\
+			hmbird_sched_change_guard_init(__rq, __p, __flags);			\
+		!__cg.done; hmbird_sched_change_guard_fini(&__cg, __flags))
+
+DECLARE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt);
+
+bool task_on_hmbird(struct task_struct *p);
+int hmbird_pre_fork(struct task_struct *p);
+int hmbird_fork(struct task_struct *p);
+void hmbird_post_fork(struct task_struct *p);
+void hmbird_cancel_fork(struct task_struct *p);
+int hmbird_check_setscheduler(struct task_struct *p, int policy);
+bool hmbird_can_stop_tick(struct rq *rq);
+void init_sched_hmbird_class(void);
+
+void hmbird_ops_exit(void);
+#define hmbird_ops_error(fmt, ...)						\
+do {										\
+	hmbird_deferred_err(HMBIRD_OPS_ERR, fmt, ##__VA_ARGS__);                      \
+	hmbird_ops_exit();				\
+} while (0)
+
+void __hmbird_notify_pick_next_task(struct rq *rq,
+				 struct task_struct *p,
+				 const struct sched_class *active);
+
+static inline unsigned long hmbird_sched_weight_to_cgroup(unsigned long weight)
+{
+	return clamp_t(unsigned long,
+			DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024),
+			CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX);
+}
+
+static inline void hmbird_notify_pick_next_task(struct rq *rq,
+						struct task_struct *p,
+						const struct sched_class *active)
+{
+	if (!hmbird_enabled())
+		return;
+#ifdef CONFIG_SMP
+	/*
+	 * Pairs with the smp_load_acquire() issued by a CPU in
+	 * kick_cpus_irq_workfn() who is waiting for this CPU to perform a
+	 * resched.
+	 */
+	smp_store_release(&get_hmbird_rq(rq)->pnt_seq, get_hmbird_rq(rq)->pnt_seq + 1);
+#endif
+	if (!static_branch_unlikely(&hmbird_ops_cpu_preempt))
+		return;
+	__hmbird_notify_pick_next_task(rq, p, active);
+}
+
+extern void hmbird_scheduler_tick(void);
+void scan_timeout(struct rq *rq);
+void hmbird_notify_sched_tick(void);
+
+static inline const struct sched_class *next_active_class(const struct sched_class *class)
+{
+	class++;
+
+	if (hmbird_enabled() && class == &fair_sched_class)
+		class++;
+	if (!hmbird_enabled() && class == &hmbird_sched_class)
+		class++;
+	return class;
+}
+
+#define for_active_class_range(class, _from, _to)				\
+	for (class = (_from); class != (_to); class = next_active_class(class))
+
+#define for_each_active_class(class)						\
+	for_active_class_range(class, __sched_class_highest, __sched_class_lowest)
+
+/*
+ * HMBIRD requires a balance() call before every pick_next_task() call including
+ * when waking up from idle.
+ */
+#define for_balance_class_range(class, prev_class, end_class)			\
+	for_active_class_range(class, (prev_class) > &hmbird_sched_class ?		\
+				&hmbird_sched_class : (prev_class), (end_class))
+
+#define MIN_CGROUP_DL_IDX (5)      /* 8ms */
+#define DEFAULT_CGROUP_DL_IDX (8)  /* 64ms */
+extern u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS];
+
+
+void __hmbird_update_idle(struct rq *rq, bool idle);
+
+static inline void hmbird_update_idle(struct rq *rq, bool idle)
+{
+	if (hmbird_enabled())
+		__hmbird_update_idle(rq, idle);
+}
+
+int hmbird_ctrl(bool enable);
+
+int hmbird_tg_online(struct task_group *tg);
+
+
+static inline u16 hmbird_task_util(struct task_struct *p)
+{
+	return (p->sched_class == &hmbird_sched_class) ? get_hmbird_ts(p)->sts.demand_scaled : 0;
+}
+u16 hmbird_cpu_util(int cpu);
+
+bool get_hmbird_ops_enabled(void);
+bool get_non_hmbird_task(void);
+void set_cpu_cluster(u64 cpu_cluster);
+#endif /*_EXT_H_*/
diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c
new file mode 100755
index 000000000000..ce05928392bf
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird.c
@@ -0,0 +1,4313 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * HMBIRD scheduler class
+ *
+ * Copyright (c) 2024 OPlus.
+ * Copyright (c) 2024 Dao Huang
+ * Copyright (c) 2024 Yuxing Wang
+ * Copyright (c) 2024 Taiyu Li
+ */
+
+#include <linux/notifier.h>
+#include <linux/panic_notifier.h>
+
+#include "slim.h"
+#include "hmbird_sched.h"
+#include "hmbird_util_track.h"
+#include <linux/sched/hmbird_proc_val.h>
+
+#define CLUSTER_SEPARATE
+
+atomic_t __hmbird_ops_enabled = ATOMIC_INIT(0);
+atomic_t non_hmbird_task;
+atomic_t hmbird_module_loaded = ATOMIC_INIT(0);
+int cgroup_ids_table[NUMS_CGROUP_KINDS];
+static int sched_prop_to_preempt_prio[HMBIRD_TASK_PROP_MAX] = {0};
+
+enum hmbird_internal_consts {
+	HMBIRD_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
+};
+
+enum hmbird_ops_enable_state {
+	HMBIRD_OPS_PREPPING,
+	HMBIRD_OPS_ENABLING,
+	HMBIRD_OPS_ENABLED,
+	HMBIRD_OPS_DISABLING,
+	HMBIRD_OPS_DISABLED,
+};
+
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
+{
+	return css ? container_of(css, struct task_group, css) : NULL;
+}
+
+static inline void check_class_changed(struct rq *rq, struct task_struct *p,
+							const struct sched_class *prev_class,
+							int oldprio)
+{
+	if (prev_class != p->sched_class) {
+		if (prev_class->switched_from)
+			prev_class->switched_from(rq, p);
+
+		p->sched_class->switched_to(rq, p);
+	} else if (oldprio != p->prio || dl_task(p))
+		p->sched_class->prio_changed(rq, p, oldprio);
+}
+
+/*
+ * hmbird_entity->ops_state
+ *
+ * Used to track the task ownership between the HMBIRD core and the BPF scheduler.
+ * State transitions look as follows:
+ *
+ * NONE -> QUEUEING -> QUEUED -> DISPATCHING
+ *   ^              |                 |
+ *   |              v                 v
+ *   \-------------------------------/
+ *
+ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call
+ * sites for explanations on the conditions being waited upon and why they are
+ * safe. Transitions out of them into NONE or QUEUED must store_release and the
+ * waiters should load_acquire.
+ *
+ * Tracking hmbird_ops_state enables hmbird core to reliably determine whether
+ * any given task can be dispatched by the BPF scheduler at all times and thus
+ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler
+ * to try to dispatch any task anytime regardless of its state as the HMBIRD core
+ * can safely reject invalid dispatches.
+ */
+enum hmbird_ops_state {
+	HMBIRD_OPSS_NONE,		/* owned by the HMBIRD core */
+	HMBIRD_OPSS_QUEUEING,	/* in transit to the BPF scheduler */
+	HMBIRD_OPSS_QUEUED,	/* owned by the BPF scheduler */
+	HMBIRD_OPSS_DISPATCHING,	/* in transit back to the HMBIRD core */
+
+	/*
+	 * QSEQ brands each QUEUED instance so that, when dispatch races
+	 * dequeue/requeue, the dispatcher can tell whether it still has a claim
+	 * on the task being dispatched.
+	 */
+	HMBIRD_OPSS_QSEQ_SHIFT	= 2,
+	HMBIRD_OPSS_STATE_MASK	= (1LLU << HMBIRD_OPSS_QSEQ_SHIFT) - 1,
+	HMBIRD_OPSS_QSEQ_MASK	= ~HMBIRD_OPSS_STATE_MASK,
+};
+
+enum switch_stat {
+	HMBIRD_DISABLED,
+	HMBIRD_SWITCH_PREP,
+	HMBIRD_RQ_SWITCH_BEGIN,
+	HMBIRD_RQ_SWITCH_DONE,
+	HMBIRD_ENABLED,
+};
+enum switch_stat curr_ss;
+
+/*
+ * During exit, a task may schedule after losing its PIDs. When disabling the
+ * BPF scheduler, we need to be able to iterate tasks in every state to
+ * guarantee system safety. Maintain a dedicated task list which contains every
+ * task between its fork and eventual free.
+ */
+DEFINE_SPINLOCK(hmbird_tasks_lock);
+static LIST_HEAD(hmbird_tasks);
+
+/* ops enable/disable */
+static struct kthread_worker *hmbird_ops_helper;
+static DEFINE_MUTEX(hmbird_ops_enable_mutex);
+DEFINE_STATIC_PERCPU_RWSEM(hmbird_fork_rwsem);
+static atomic_t hmbird_ops_enable_state_var = ATOMIC_INIT(HMBIRD_OPS_DISABLED);
+
+static bool hmbird_warned_zero_slice;
+enum hmbird_switch_type sw_type;
+static int skip_num[MAX_GLOBAL_DSQS];
+static int big_distribute_mask_prev;
+static int little_distribute_mask_prev;
+
+DEFINE_STATIC_KEY_FALSE(hmbird_ops_cpu_preempt);
+
+static atomic64_t hmbird_nr_rejected = ATOMIC64_INIT(0);
+
+/*
+ * The maximum amount of time in jiffies that a task may be runnable without
+ * being scheduled on a CPU. If this timeout is exceeded, it will trigger
+ * hmbird_ops_error().
+ */
+unsigned long hmbird_watchdog_timeout;
+
+/*
+ * The last time the delayed work was run. This delayed work relies on
+ * ksoftirqd being able to run to service timer interrupts, so it's possible
+ * that this work itself could get wedged. To account for this, we check that
+ * it's not stalled in the timer tick, and trigger an error if it is.
+ */
+unsigned long hmbird_watchdog_timestamp = INITIAL_JIFFIES;
+
+static struct delayed_work hmbird_watchdog_work;
+static struct work_struct hmbird_err_exit_work;
+
+/* idle tracking */
+#ifdef CONFIG_SMP
+#ifdef CONFIG_CPUMASK_OFFSTACK
+#define CL_ALIGNED_IF_ONSTACK
+#else
+#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp
+#endif
+
+static struct {
+	cpumask_var_t cpu;
+	cpumask_var_t smt;
+} idle_masks CL_ALIGNED_IF_ONSTACK;
+
+static bool __cacheline_aligned_in_smp hmbird_has_idle_cpus;
+#endif	/* CONFIG_SMP */
+
+/* dispatch queues */
+static struct hmbird_dispatch_q __cacheline_aligned_in_smp hmbird_dsq_global;
+
+u32 HMBIRD_BPF_DSQS_DEADLINE[MAX_GLOBAL_DSQS] = {0, 1, 2, 4, 6, 8, 16, 32, 64, 128};
+u32 pcp_dsq_deadline = 20;
+static struct hmbird_dispatch_q __cacheline_aligned_in_smp gdsqs[MAX_GLOBAL_DSQS];
+static DEFINE_PER_CPU(struct hmbird_dispatch_q, pcp_ldsq);
+
+static u64 max_hmbird_dsq_internal_id;
+
+/* a dsq idx, whether task push to little domain cpu or bit domain cpu*/
+#define CLUSTER_SEPARATE_IDX	(8)
+#define GDSQS_ID_BASE		(3)
+#define UX_COMPATIBLE_IDX	(4)
+#define NON_PERIOD_START	(5)
+#define NON_PERIOD_END		(MAX_GLOBAL_DSQS)
+#define CREATE_DSQ_LEVEL_WITHIN	(1)
+
+struct hmbird_sched_info {
+	spinlock_t lock;
+	int curr_idx[2];
+	int rtime[MAX_GLOBAL_DSQS];
+};
+
+struct pcp_sched_info {
+	s64 pcp_seq;
+	int rtime;
+	bool pcp_round;
+};
+
+/*
+ * pcp_info may rw by another cpu.
+ * protected by rq->lock.
+ */
+atomic64_t pcp_dsq_round;
+static DEFINE_PER_CPU(struct pcp_sched_info, pcp_info);
+
+struct md_info_t *md_info;
+
+static int b_rescue_l, l_rescue_b;
+static struct hmbird_sched_info sinfo;
+
+static unsigned long pcp_dsq_quota __read_mostly = 3 * NSEC_PER_MSEC;
+static unsigned long dsq_quota[MAX_GLOBAL_DSQS] = {
+					0, 0, 0, 0, 0,
+					32 * NSEC_PER_MSEC,
+					20 * NSEC_PER_MSEC,
+					14 * NSEC_PER_MSEC,
+					8 * NSEC_PER_MSEC,
+					6 * NSEC_PER_MSEC
+};
+
+struct cluster_ctx {
+	/* cpu-dsq map must within [lower, upper) */
+	int upper;
+	int lower;
+	int tidx;
+};
+
+enum stat_items {
+	GLOBAL_STAT,
+	CPU_ALLOW_FAIL,
+	RT_CNT,
+	KEY_TASK_CNT,
+	SWITCH_IDX,
+	TIMEOUT_CNT,
+
+	TOTAL_DSP_CNT,
+	MOVE_RQ_CNT,
+	SELECT_CPU,
+
+	DWORD_STAT_END = SELECT_CPU,
+
+	GDSQ_CNT,
+	ERR_IDX,
+	PCP_TIMEOUT_CNT,
+	PCP_LDSQ_CNT,
+	PCP_ENQL_CNT,
+
+	MAX_ITEMS,
+};
+static DEFINE_SPINLOCK(stats_lock);
+static char *stats_str[MAX_ITEMS] = {
+	"global stat", "cpu_allow_fail", "rt_cnt", "key_task_cnt",
+	"switch_idx", "timeout_cnt", "total_dsp_cnt", "move_rq_cnt",
+	"select_cpu", "gdsq_cnt", "err_idx", "pcp_timeout_cnt",
+	"pcp_ldsq_cnt", "pcp_enql_cnt"
+};
+
+
+struct stats_struct {
+	u64 global_stat[2];
+	u64 cpu_allow_fail[2];
+	u64 rt_cnt[2];
+	u64 key_task_cnt[2];
+	u64 switch_idx[2];
+	u64 timeout_cnt[2];
+
+	u64 total_dsp_cnt[2];
+	u64 move_rq_cnt[2];
+	u64 select_cpu[2];
+
+	u64 gdsq_count[MAX_GLOBAL_DSQS][2];
+	u64 err_idx[5];
+	u64 pcp_timeout_cnt[NR_CPUS];
+	u64 pcp_ldsq_count[NR_CPUS][2];
+	u64 pcp_enql_cnt[NR_CPUS];
+} stats_data;
+
+static struct {
+	cpumask_var_t ex_free;
+	cpumask_var_t exclusive;
+	cpumask_var_t partial;
+	cpumask_var_t big;
+	cpumask_var_t little;
+} iso_masks __read_mostly;
+
+/*
+ * Need more synchronization for these two variables?
+ * I choose not to.
+ */
+static int l_need_rescue, b_need_rescue;
+
+#define HMBIRD_FATAL_INFO_FN(type, fmt, args...)				\
+{										\
+	char buf[MAX_FATAL_INFO];						\
+										\
+	scnprintf(buf, MAX_FATAL_INFO, fmt, ##args);				\
+	hmbird_err(HMBIRD_OPS_ERR, "type(%d) %s\n", type, buf);			\
+	trace_hmbird_fatal_info((unsigned int)type, READ_ONCE(partial_enable),	\
+		READ_ONCE(l_need_rescue), READ_ONCE(b_need_rescue), buf);	\
+	queue_work(system_unbound_wq, &hmbird_err_exit_work);			\
+}										\
+
+static bool cpu_same_cluster_stat(struct task_struct *p, struct rq *rq1, struct rq *rq2)
+{
+	int c1, c2;
+	struct cpumask mask = {.bits = {0}};
+	struct cpumask tmp = {.bits = {0}};
+
+	if (!slim_stats)
+		return false;
+
+	if (!rq1 || !rq2)
+		return false;
+
+	c1 = cpu_of(rq1);
+	c2 = cpu_of(rq2);
+	cpumask_set_cpu(c1, &mask);
+	cpumask_set_cpu(c2, &mask);
+
+	if (cpumask_and(&tmp, iso_masks.little, &mask))
+		if (cpumask_equal(&tmp, &mask))
+			return true;
+
+	if (cpumask_and(&tmp, iso_masks.big, &mask))
+		if (cpumask_equal(&tmp, &mask))
+			return true;
+
+	if (cpumask_and(&tmp, iso_masks.partial, &mask))
+		if (cpumask_equal(&tmp, &mask))
+			return true;
+
+	return false;
+}
+
+static void slim_stats_record(enum stat_items item, int idx, int dsq_id, int cpu)
+{
+	unsigned long flags;
+	u64 *pval;
+	u64 *pbase = (u64 *)&stats_data;
+
+	if (!slim_stats)
+		return;
+
+	switch (item) {
+	case GLOBAL_STAT:
+		fallthrough;
+	case CPU_ALLOW_FAIL:
+		fallthrough;
+	case RT_CNT:
+		fallthrough;
+	case KEY_TASK_CNT:
+		fallthrough;
+	case SWITCH_IDX:
+		fallthrough;
+	case TIMEOUT_CNT:
+		fallthrough;
+	case TOTAL_DSP_CNT:
+		fallthrough;
+	case MOVE_RQ_CNT:
+		fallthrough;
+	case SELECT_CPU:
+		pval = pbase + item * 2 + idx;
+		break;
+	case GDSQ_CNT:
+		pval = &stats_data.gdsq_count[dsq_id][idx];
+		break;
+	case ERR_IDX:
+		pval = &stats_data.err_idx[idx];
+		break;
+	case PCP_TIMEOUT_CNT:
+		pval = &stats_data.pcp_timeout_cnt[cpu];
+		break;
+	case PCP_LDSQ_CNT:
+		pval = &stats_data.pcp_ldsq_count[cpu][idx];
+		break;
+	case PCP_ENQL_CNT:
+		pval = &stats_data.pcp_enql_cnt[cpu];
+		break;
+	default:
+		return;
+	}
+
+	spin_lock_irqsave(&stats_lock, flags);
+	*pval += 1;
+	spin_unlock_irqrestore(&stats_lock, flags);
+}
+
+static inline bool handle_ret(int ret, int *idx, int len)
+{
+	if (ret < 0 || ret >= len - *idx)
+		return true;
+	*idx += ret;
+	return false;
+}
+
+#define PRINT_INTV	(5 * HZ)
+void stats_print(char *buf, int len)
+{
+	int idx = 0, i, j, ret;
+	int item = 0;
+	u64 *pval;
+	u64 *pbase = (u64 *)&stats_data;
+
+	ret = snprintf(&buf[idx], len - idx, "-------------schedinfo stats :---------------\n");
+	if (handle_ret(ret, &idx, len))
+		return;
+	for (item = 0; item < MAX_ITEMS; item++) {
+		if (item <= DWORD_STAT_END) {
+			pval = pbase + item * 2;
+			ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu\n",
+					stats_str[item], pval[0],  pval[1]);
+			if (handle_ret(ret, &idx, len))
+				return;
+		} else if (item == GDSQ_CNT) {
+			for (j = 0; j < MAX_GLOBAL_DSQS; j++) {
+				pval = (u64 *)&stats_data.gdsq_count[j];
+				ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu, %llu\n",
+						stats_str[item], j, pval[0], pval[1]);
+				if (handle_ret(ret, &idx, len))
+					return;
+			}
+		} else if (item == ERR_IDX) {
+			pval = (u64 *)&stats_data.err_idx;
+			ret = snprintf(&buf[idx], len - idx, "%s:%llu, %llu, %llu, %llu, %llu\n",
+						stats_str[item], pval[0],
+						pval[1], pval[2], pval[3], pval[4]);
+			if (handle_ret(ret, &idx, len))
+				return;
+		} else if (item == PCP_TIMEOUT_CNT) {
+			for (j = 0; j < nr_cpu_ids; j++) {
+				pval = (u64 *)&stats_data.pcp_timeout_cnt[j];
+				ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n",
+							stats_str[item], j, *pval);
+				if (handle_ret(ret, &idx, len))
+					return;
+			}
+		} else if (item == PCP_LDSQ_CNT) {
+			for (j = 0; j < nr_cpu_ids; j++) {
+				pval = (u64 *)&stats_data.pcp_ldsq_count[j];
+				ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu,%llu\n",
+						stats_str[item], j, pval[0], pval[1]);
+				if (handle_ret(ret, &idx, len))
+					return;
+			}
+		} else if (item == PCP_ENQL_CNT) {
+			for (j = 0; j < nr_cpu_ids; j++) {
+				pval = (u64 *)&stats_data.pcp_enql_cnt[j];
+				ret = snprintf(&buf[idx], len - idx, "%s[%d]:%llu\n",
+						stats_str[item], j, *pval);
+				if (handle_ret(ret, &idx, len))
+					return;
+			}
+		}
+	}
+
+	if (!md_info) {
+		buf[idx] = '\0';
+		return;
+	}
+
+	ret = snprintf(&buf[idx], len - idx, "\n\n------------minidump stats :---------------\n");
+	if (handle_ret(ret, &idx, len))
+		return;
+
+	for (i = 0; i < MAX_SWITCHS; i++) {
+		ret = snprintf(&buf[idx], len - idx,
+				"sw_rec[%d] = %llu %llu %llu %llu\n", i,
+				md_info->kern_dump.sw_rec[i].switch_at,
+				md_info->kern_dump.sw_rec[i].is_success,
+				md_info->kern_dump.sw_rec[i].end_state,
+				md_info->kern_dump.sw_rec[i].switch_reason);
+		if (handle_ret(ret, &idx, len))
+			return;
+	}
+	ret = snprintf(&buf[idx], len - idx, "sw_idx = %llu\n", md_info->kern_dump.sw_idx);
+	if (handle_ret(ret, &idx, len))
+		return;
+
+	for (i = 0; i < MAX_EXCEP_ID; i++) {
+		ret = snprintf(&buf[idx], len - idx,
+				"excep[%d] = %llu %llu %llu %llu %llu\n", i,
+				md_info->kern_dump.excep_rec[i][0],
+				md_info->kern_dump.excep_rec[i][1],
+				md_info->kern_dump.excep_rec[i][2],
+				md_info->kern_dump.excep_rec[i][3],
+				md_info->kern_dump.excep_rec[i][4]);
+		if (handle_ret(ret, &idx, len))
+			return;
+
+		ret = snprintf(&buf[idx], len - idx, "excep_idx[%d] = %llu\n",
+					i, md_info->kern_dump.excep_idx[i]);
+		if (handle_ret(ret, &idx, len))
+			return;
+	}
+
+	buf[idx] = '\0';
+}
+
+enum cpu_type {
+	LITTLE,
+	BIG,
+	PARTIAL,
+	EXCLUSIVE,
+	EX_FREE,
+	INVALID
+};
+
+enum dsq_type {
+	GLOBAL_DSQ,
+	PCP_DSQ,
+	OTHER,
+	MAX_DSQ_TYPE,
+};
+
+static enum cpu_type cpu_cluster(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (get_hmbird_rq(rq)->exclusive) {
+		return EXCLUSIVE;
+	} else {
+		if (cpumask_test_cpu(cpu, iso_masks.little))
+			return LITTLE;
+		else if (cpumask_test_cpu(cpu, iso_masks.big))
+			return BIG;
+		else if (cpumask_test_cpu(cpu, iso_masks.partial))
+			return PARTIAL;
+		else if (cpumask_test_cpu(cpu, iso_masks.exclusive))
+			return EXCLUSIVE;
+		else if (cpumask_test_cpu(cpu, iso_masks.ex_free))
+			return EX_FREE;
+	}
+	return INVALID;
+}
+
+static enum dsq_type get_dsq_type(struct hmbird_dispatch_q *dsq)
+{
+	if (!dsq)
+		return OTHER;
+
+	if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) &&
+		((dsq->id & 0xff) >= GDSQS_ID_BASE) &&
+		((dsq->id & 0xff) < MAX_GLOBAL_DSQS))
+		return GLOBAL_DSQ;
+
+	if ((dsq->id & HMBIRD_DSQ_FLAG_BUILTIN) &&
+		((dsq->id & 0xff) >= MAX_GLOBAL_DSQS) &&
+		((dsq->id & 0xff) < max_hmbird_dsq_internal_id))
+		return PCP_DSQ;
+
+	return OTHER;
+}
+
+static int dsq_id_to_internal(struct hmbird_dispatch_q *dsq)
+{
+	enum dsq_type type;
+
+	type = get_dsq_type(dsq);
+	switch (type) {
+	case GLOBAL_DSQ:
+	case PCP_DSQ:
+		return (dsq->id & 0xff) - GDSQS_ID_BASE;
+	default:
+		return -1;
+	}
+	return -1;
+}
+
+static void update_cpus_idle(bool set, struct cpumask *mask)
+{
+	int cpu;
+	struct rq *rq;
+
+	if (set) {
+		for_each_cpu(cpu, mask) {
+			rq = cpu_rq(cpu);
+			if (is_idle_task(rq->curr))
+				cpumask_set_cpu(cpu, idle_masks.cpu);
+		}
+	} else
+		cpumask_andnot(idle_masks.cpu, idle_masks.cpu, mask);
+}
+
+static void set_partial_status(bool enable, bool little, bool big)
+{
+	WRITE_ONCE(partial_enable, enable);
+	WRITE_ONCE(l_need_rescue, little);
+	WRITE_ONCE(b_need_rescue, big);
+}
+
+static bool is_little_need_rescue(void)
+{
+	return READ_ONCE(l_need_rescue);
+}
+
+static bool is_big_need_rescue(void)
+{
+	return READ_ONCE(b_need_rescue);
+}
+
+static bool is_partial_enabled(void)
+{
+	return READ_ONCE(partial_enable);
+}
+
+static bool is_partial_cpu(int cpu)
+{
+	return cpumask_test_cpu(cpu, iso_masks.partial);
+}
+
+static void set_iso_par_free(bool enable)
+{
+	WRITE_ONCE(isolate_ctrl, enable);
+}
+
+static bool is_iso_par_free(void)
+{
+	return READ_ONCE(isolate_ctrl);
+}
+
+static bool skip_update_idle(void)
+{
+	int cpu = smp_processor_id();
+	enum cpu_type type = cpu_cluster(cpu);
+
+	if ((type == EXCLUSIVE && !is_iso_par_free()) ||
+		/* partial enable may changed during idle, it doesn't matter. */
+		(!is_partial_enabled() && type == PARTIAL))
+		return true;
+
+	return false;
+}
+
+static void init_isolate_cpus(void)
+{
+	WARN_ON(!alloc_cpumask_var(&iso_masks.ex_free, GFP_KERNEL));
+	WARN_ON(!alloc_cpumask_var(&iso_masks.partial, GFP_KERNEL));
+	WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL));
+	WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL));
+	WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL));
+	cpumask_set_cpu(0, iso_masks.big);
+	cpumask_set_cpu(1, iso_masks.big);
+	cpumask_set_cpu(2, iso_masks.big);
+	cpumask_set_cpu(3, iso_masks.big);
+	cpumask_set_cpu(4, iso_masks.big);
+	cpumask_set_cpu(5, iso_masks.ex_free);
+	cpumask_set_cpu(6, iso_masks.partial);
+	cpumask_set_cpu(7, iso_masks.ex_free);
+}
+
+extern spinlock_t css_set_lock;
+
+static struct cgroup *cgroup_ancestor_l1(struct cgroup *cgrp)
+{
+	int i;
+	struct cgroup *anc;
+
+	spin_lock_irq(&css_set_lock);
+	for (i = 0; i < cgrp->level; i++) {
+		anc = cgrp->ancestors[i];
+		if (anc->level != CREATE_DSQ_LEVEL_WITHIN)
+			continue;
+		cgroup_get(anc);
+		spin_unlock_irq(&css_set_lock);
+		return anc;
+	}
+	spin_unlock_irq(&css_set_lock);
+	hmbird_err(NO_CGROUP_L1, "<fatal>:error cgroup = %s\n", cgrp->kn->name);
+	return NULL;
+}
+
+#define PCP_IDX_BIT    (1 << 31)
+
+static bool is_pcp_rt(struct task_struct *p)
+{
+	return rt_prio(p->prio) && (p->nr_cpus_allowed == 1);
+}
+
+static bool is_pcp_idx(int idx)
+{
+	return idx >= MAX_GLOBAL_DSQS;
+}
+
+static bool is_critical_system_task(struct task_struct *p)
+{
+	int sp_dl = get_hmbird_ts(p)->sched_prop & SCHED_PROP_DEADLINE_MASK;
+
+	return (p->prio < (MAX_RT_PRIO >> 1) &&
+		(sp_dl < SCHED_PROP_DEADLINE_LEVEL3));
+}
+
+#define ISOLATE_TASK_TOP_BIT	(1 << 17)
+#define PIPELINE_TASK_TOP_BIT	(1 << 9)
+static bool is_pipeline_task(struct task_struct *p)
+{
+	return (get_top_task_prop(p) & PIPELINE_TASK_TOP_BIT);
+}
+
+static bool is_isolate_task(struct task_struct *p)
+{
+	return (get_top_task_prop(p) & ISOLATE_TASK_TOP_BIT);
+}
+
+static bool is_critical_app_task_without_isolate(struct task_struct *p)
+{
+	return task_is_top_task(p) && !is_isolate_task(p);
+}
+
+static int find_idx_from_task(struct task_struct *p)
+{
+	int idx, cpu;
+	int sp_dl;
+	struct task_group *tg = p->sched_task_group;
+
+	if (p->nr_cpus_allowed == 1 || is_migration_disabled(p)) {
+		cpu = cpumask_any(p->cpus_ptr);
+		idx = cpu + MAX_GLOBAL_DSQS;
+		return idx;
+	}
+
+	if (is_critical_system_task(p)) {
+		idx = SCHED_PROP_DEADLINE_LEVEL0;
+		goto done;
+	}
+
+	if (is_critical_app_task_without_isolate(p)) {
+		idx = SCHED_PROP_DEADLINE_LEVEL1;
+		goto done;
+	}
+
+	sp_dl = hmbird_get_dsq_id(p);
+	if (sp_dl) {
+		idx = sp_dl;
+		goto done;
+	}
+
+	if (rt_prio(p->prio)) {
+		idx = SCHED_PROP_DEADLINE_LEVEL3;
+		goto done;
+	}
+
+	if (tg && tg->css.cgroup && tg->css.cgroup->kn) {
+		if (likely(tg->css.cgroup->kn->id >= 0 &&
+			tg->css.cgroup->kn->id < NUMS_CGROUP_KINDS))
+			idx = cgroup_ids_table[tg->css.cgroup->kn->id];
+		else
+			idx = DEFAULT_CGROUP_DL_IDX;
+	} else
+		idx = DEFAULT_CGROUP_DL_IDX;
+
+done:
+	if (idx < 0 || idx >= MAX_GLOBAL_DSQS) {
+		hmbird_err(DSQ_ID_ERR, "<slim_sched><error> : idx error, idx = %d-----\n", idx);
+		idx = DEFAULT_CGROUP_DL_IDX;
+	}
+	return idx;
+}
+
+static struct hmbird_dispatch_q *find_dsq_from_task(struct task_struct *p)
+{
+	int idx;
+	unsigned long flags;
+	struct hmbird_dispatch_q *dsq;
+
+	if (!p)
+		return NULL;
+
+	idx = find_idx_from_task(p);
+	if (is_pcp_idx(idx)) {
+		idx -= MAX_GLOBAL_DSQS;
+		dsq = &per_cpu(pcp_ldsq, idx);
+		get_hmbird_ts(p)->gdsq_idx = dsq_id_to_internal(dsq);
+		slim_stats_record(PCP_LDSQ_CNT, 0, 0, idx);
+	} else {
+		dsq = &gdsqs[idx];
+		get_hmbird_ts(p)->gdsq_idx = idx;
+		slim_stats_record(GDSQ_CNT, 0, idx, 0);
+	}
+
+	raw_spin_lock_irqsave(&dsq->lock, flags);
+	if (list_empty(&dsq->fifo))
+		dsq->last_consume_at = jiffies;
+
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+
+	return dsq;
+}
+
+
+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf,
+						struct hmbird_dispatch_q *dsq);
+
+static void set_partial_rescue(bool p_state, bool l_over, bool b_over)
+{
+	set_partial_status(p_state, l_over, b_over);
+	update_cpus_idle(p_state, iso_masks.partial);
+	hmbird_internal_systrace("C|9999|partial_enable|%d\n", is_partial_enabled());
+	hmbird_internal_systrace("C|9999|l_need_rescue|%d\n", is_little_need_rescue());
+	hmbird_internal_systrace("C|9999|b_need_rescue|%d\n", is_big_need_rescue());
+}
+
+static void free_isocpu(bool enable)
+{
+	set_iso_par_free(enable);
+	update_cpus_idle(enable, iso_masks.ex_free);
+	hmbird_internal_systrace("C|9999|free_iso|%d\n", is_iso_par_free());
+}
+
+inline u64 get_hmbird_cpu_util(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (!get_hmbird_rq(rq)->prev_runnable_sum_fixed)
+		return 0;
+	u64 prev_runnable_sum_fixed = *(u64 *)(get_hmbird_rq(rq)->prev_runnable_sum_fixed);
+	u32 prev_window_size = *(u32 *)(get_hmbird_rq(rq)->prev_window_size);
+
+	do_div(prev_runnable_sum_fixed, prev_window_size >> SCHED_CAPACITY_SHIFT);
+
+	return prev_runnable_sum_fixed;
+}
+
+static inline unsigned int get_scaling_max_freq(unsigned int cpu)
+{
+	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+
+	return (policy == NULL) ? 0 : policy->max;
+}
+
+static inline unsigned int get_cpuinfo_max_freq(unsigned int cpu)
+{
+	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+
+	return (policy == NULL) ? 0 : policy->cpuinfo.max_freq;
+}
+
+static u64 get_cpus_max_util(struct cpumask *mask)
+{
+	int cpu;
+	u64 max = 0;
+	u64 util, ratio;
+	unsigned long effective_cap = 0;
+	for_each_cpu(cpu, mask) {
+		if (slim_walt_ctrl)
+			slim_get_cpu_util(cpu, &util);
+		else
+			util = get_hmbird_cpu_util(cpu);
+
+		/* if max freq is 0, effective_cap use arch_scale_cpu_capacity*/
+		if (unlikely(!get_scaling_max_freq(cpu) || !get_cpuinfo_max_freq(cpu)))
+			effective_cap = arch_scale_cpu_capacity(cpu);
+		else
+			effective_cap = arch_scale_cpu_capacity(cpu) *
+					get_scaling_max_freq(cpu) / get_cpuinfo_max_freq(cpu);
+
+		ratio = util * 100 / effective_cap;
+		hmbird_info_systrace("C|9999|Cpu%d_util|%llu\n", cpu, util);
+		hmbird_info_systrace("C|9999|Cpu%d_cap|%llu\n",
+				cpu, (u64)arch_scale_cpu_capacity(cpu));
+		hmbird_info_systrace("C|9999|Cpu%d_effective_cap|%llu\n",
+				cpu, (u64)effective_cap);
+
+		if (ratio > 100)
+			ratio = 100;
+
+		if (ratio > max)
+			max = ratio;
+	}
+	return max;
+}
+
+static bool cluster_need_rescue(struct cpumask *mask, int hres)
+{
+	return get_cpus_max_util(mask) > hres;
+}
+
+void partial_dynamic_ctrl(void)
+{
+	u64 lmax = 0, bmax = 0;
+	bool l_over, l_under, b_over, b_under;
+	static bool last_l_over, last_b_over;
+	static unsigned long last_check;
+
+	/* Check partial every jiffies. need lock? */
+	if (time_before_eq(jiffies, READ_ONCE(last_check)))
+		return;
+
+	WRITE_ONCE(last_check, jiffies);
+
+	lmax = get_cpus_max_util(iso_masks.little);
+	l_over = lmax > parctrl_high_ratio_l;
+	l_under = lmax < parctrl_low_ratio_l;
+
+	bmax = get_cpus_max_util(iso_masks.big);
+	b_over = bmax > parctrl_high_ratio;
+	b_under = bmax < parctrl_low_ratio;
+
+	if (is_partial_enabled() && (l_over || b_over)) {
+		if (last_l_over != l_over || last_b_over != b_over)
+			set_partial_rescue(true, l_over, b_over);
+	} else if (!is_partial_enabled() && (l_over || b_over)) {
+		set_partial_rescue(true, l_over, b_over);
+	} else if (is_partial_enabled() && l_under && b_under) {
+		set_partial_rescue(false, false, false);
+	}
+	last_l_over = l_over;
+	last_b_over = b_over;
+
+	if (is_partial_enabled()) {
+		bmax = get_cpus_max_util(iso_masks.partial);
+		if (!is_iso_par_free() && bmax > isoctrl_high_ratio) {
+			hmbird_info_trace("<par>partial max = %llu\n", bmax);
+			free_isocpu(true);
+		} else if (is_iso_par_free() && bmax < isoctrl_low_ratio)
+			free_isocpu(false);
+	} else if (is_iso_par_free()) {
+		free_isocpu(false);
+	}
+}
+
+static inline void slim_trace_show_cpu_consume_dsq_idx(unsigned int cpu, unsigned int idx)
+{
+	hmbird_internal_systrace("C|9999|Cpu%d_dsq_id|%d\n", cpu, idx);
+}
+
+static int consume_target_dsq(struct rq *rq, struct rq_flags *rf, unsigned int idx)
+{
+	if (idx < 0 || idx >= MAX_GLOBAL_DSQS)
+		return 0;
+
+	if (consume_dispatch_q(rq, rf, &gdsqs[idx])) {
+		slim_stats_record(GDSQ_CNT, 1, idx, 0);
+		return 1;
+	}
+	return 0;
+}
+
+static int consume_period_dsq(struct rq *rq, struct rq_flags *rf)
+{
+	int i;
+
+	for (i = 0; i < UX_COMPATIBLE_IDX; i++) {
+		if (consume_dispatch_q(rq, rf, &gdsqs[i])) {
+			slim_stats_record(GDSQ_CNT, 1, i, 0);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static int consume_ux_dsq(struct rq *rq, struct rq_flags *rf)
+{
+	if (consume_dispatch_q(rq, rf, &gdsqs[UX_COMPATIBLE_IDX])) {
+		slim_stats_record(GDSQ_CNT, 1, UX_COMPATIBLE_IDX, 0);
+		return 1;
+	}
+
+	return 0;
+}
+
+static void update_timeout_stats(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline)
+{
+	struct hmbird_entity *entity;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&dsq->lock, flags);
+	if (list_empty(&dsq->fifo))
+		goto clear_timeout;
+
+	entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo);
+	if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline)))
+		goto clear_timeout;
+
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+	hmbird_info_trace(
+				"dsq[%d] still timeout task-%s, jiffies = %lu, deadline = %lu, runnable at = %lu\n",
+				dsq_id_to_internal(dsq), entity->task->comm,
+				jiffies, msecs_to_jiffies(deadline), entity->runnable_at);
+	hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 1);
+	return;
+
+clear_timeout:
+	hmbird_info_trace("dsq[%d] clear timeout\n",
+				dsq_id_to_internal(dsq));
+	hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id_to_internal(dsq), 0);
+	dsq->is_timeout = false;
+	slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu_of(rq));
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+}
+
+static void systrace_output_rtime_state(struct hmbird_dispatch_q *dsq, int rtime)
+{
+	hmbird_info_systrace("C|9999|dsq%d_rtime|%d\n", dsq_id_to_internal(dsq), rtime);
+}
+
+static int consume_pcp_dsq(struct rq *rq, struct rq_flags *rf, bool any)
+{
+	bool is_timeout;
+	int cpu = cpu_of(rq);
+	unsigned long flags;
+	struct hmbird_dispatch_q *dsq = &per_cpu(pcp_ldsq, cpu);
+
+	raw_spin_lock_irqsave(&dsq->lock, flags);
+	is_timeout = dsq->is_timeout;
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+
+	/*
+	 * dsq->is_timeout may change here, let it be.
+	 * it won't cause serious problems.
+	 * the same for consume_dispatch_q later.
+	 */
+	if (!is_timeout && !any)
+		return 0;
+
+	if (consume_dispatch_q(rq, rf, dsq)) {
+		if (is_timeout) {
+			hmbird_info_trace("dsq[%d] consume a pcp timeout task\n",
+						dsq_id_to_internal(dsq));
+			update_timeout_stats(rq, dsq, pcp_dsq_deadline);
+			slim_stats_record(PCP_TIMEOUT_CNT, 0, 0, cpu);
+		}
+		slim_stats_record(PCP_LDSQ_CNT, 1, 0, cpu);
+		return 1;
+	}
+	/*
+	 * No pcp task, clear quota.
+	 */
+	if (any) {
+		if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) {
+			per_cpu(pcp_info, cpu).rtime = 0;
+			per_cpu(pcp_info, cpu).pcp_round = false;
+			hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false);
+			systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu),
+					per_cpu(pcp_info, cpu).rtime);
+		}
+	}
+	return 0;
+}
+
+static int check_pcp_dsq_round(struct rq *rq, struct rq_flags *rf)
+{
+	if (per_cpu(pcp_info, cpu_of(rq)).pcp_round) {
+		if (consume_pcp_dsq(rq, rf, true))
+			return 1;
+	}
+	return 0;
+}
+
+static int check_non_period_dsq_phase(struct rq *rq, struct rq_flags *rf,
+									int tmp, int cidx, int tidx)
+{
+	unsigned long flags;
+
+	if (consume_dispatch_q(rq, rf, &gdsqs[tmp])) {
+		if (tmp != cidx) {
+			spin_lock(&sinfo.lock);
+			sinfo.curr_idx[tidx] = tmp;
+			spin_unlock(&sinfo.lock);
+			hmbird_info_systrace("C|9999|cidx_%d|%d\n", tidx, sinfo.curr_idx[tidx]);
+			slim_stats_record(SWITCH_IDX, 0, 0, 0);
+		}
+		slim_stats_record(GDSQ_CNT, 1, tmp, 0);
+
+		raw_spin_lock_irqsave(&gdsqs[tmp].lock, flags);
+		gdsqs[tmp].last_consume_at = jiffies;
+		raw_spin_unlock_irqrestore(&gdsqs[tmp].lock, flags);
+		return 1;
+	}
+	return 0;
+
+}
+
+static int get_cidx(struct cluster_ctx *ctx)
+{
+	int cidx;
+
+	spin_lock(&sinfo.lock);
+	cidx = sinfo.curr_idx[ctx->tidx];
+	if (cidx < ctx->lower || cidx >= ctx->upper) {
+		sinfo.curr_idx[ctx->tidx] = ctx->lower;
+		hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx->tidx, sinfo.curr_idx[ctx->tidx]);
+		slim_stats_record(ERR_IDX, ctx->tidx + 3, 0, 0);
+		cidx = sinfo.curr_idx[ctx->tidx];
+	}
+	spin_unlock(&sinfo.lock);
+
+	return cidx;
+}
+
+static int gen_cluster_ctx_separate(struct cluster_ctx *ctx, enum cpu_type type)
+{
+	switch (type) {
+	case PARTIAL:
+		if (!is_partial_enabled())
+			return -1;
+		fallthrough;
+	case BIG:
+		ctx->lower = NON_PERIOD_START;
+		ctx->upper = CLUSTER_SEPARATE_IDX;
+		if (!cpumask_available(iso_masks.little) || cpumask_empty(iso_masks.little)) {
+			pr_debug("<hmbird sched> %s iso_masks.little first is %d\n",
+					__func__, cpumask_first(iso_masks.little));
+			ctx->upper = NON_PERIOD_END;
+		}
+		ctx->tidx = 0;
+		break;
+	case LITTLE:
+		ctx->lower = CLUSTER_SEPARATE_IDX;
+		ctx->upper = NON_PERIOD_END;
+		if (!cpumask_available(iso_masks.big) || cpumask_empty(iso_masks.big)) {
+			pr_debug("<hmbird sched> %s iso_masks.big first is %d",
+					__func__, cpumask_first(iso_masks.big));
+			ctx->lower = NON_PERIOD_START;
+		}
+		ctx->tidx = 1;
+		break;
+	default:
+		hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n");
+		return -1;
+	}
+	return 0;
+}
+
+static int gen_cluster_ctx_common(struct cluster_ctx *ctx, enum cpu_type type)
+{
+	switch (type) {
+	case PARTIAL:
+		if (!is_partial_enabled())
+			return -1;
+		fallthrough;
+	case BIG:
+	case LITTLE:
+		ctx->lower = NON_PERIOD_START;
+		ctx->upper = NON_PERIOD_END;
+		ctx->tidx = 0;
+		break;
+	default:
+		hmbird_deferred_err(CPU_NO_MASK, "can't find cpu cluster\n");
+		return -1;
+	}
+	return 0;
+}
+
+static int gen_cluster_ctx(struct cluster_ctx *ctx, enum cpu_type type)
+{
+	if (cluster_separate) {
+		return gen_cluster_ctx_separate(ctx, type);
+	} else {
+		return gen_cluster_ctx_common(ctx, type);
+	}
+}
+
+static int consume_timeout_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type)
+{
+	int i;
+	bool is_timeout;
+	unsigned long flags;
+	struct cluster_ctx ctx;
+
+	if (gen_cluster_ctx(&ctx, type))
+		return 0;
+
+	/* Third param <false> means only consume timeout pcp dsq. */
+	if (consume_pcp_dsq(rq, rf, false))
+		return 1;
+
+	for (i = ctx.lower; i < ctx.upper; i++) {
+		raw_spin_lock_irqsave(&gdsqs[i].lock, flags);
+		is_timeout = gdsqs[i].is_timeout;
+		raw_spin_unlock_irqrestore(&gdsqs[i].lock, flags);
+		/* gdsqs[i].is_timeout may change here, let it be... */
+		if (is_timeout) {
+			/*
+			 * consume_dispatch_q will acquire dsq-lock,
+			 * So cannot keep lock here, annoy enough.
+			 * may rewrite a consume_dispatch_q_locked, TODO.
+			 */
+			if (consume_dispatch_q(rq, rf, &gdsqs[i])) {
+				hmbird_info_trace("dsq[%d] consume a timeout task\n", i);
+				slim_stats_record(TIMEOUT_CNT, ctx.tidx, 0, 0);
+				update_timeout_stats(rq, &gdsqs[i], HMBIRD_BPF_DSQS_DEADLINE[i]);
+				return 1;
+			}
+		}
+	}
+	return 0;
+}
+
+static int consume_non_period_dsq(struct rq *rq, struct rq_flags *rf, enum cpu_type type)
+{
+	struct cluster_ctx ctx;
+	int cidx;
+	int tmp;
+
+	if (gen_cluster_ctx(&ctx, type))
+		return 0;
+
+	cidx = get_cidx(&ctx);
+	tmp = cidx;
+	do {
+		if (check_pcp_dsq_round(rq, rf))
+			return 1;
+		if (check_non_period_dsq_phase(rq, rf, tmp, cidx, ctx.tidx))
+			return 1;
+		spin_lock(&sinfo.lock);
+		sinfo.rtime[tmp] = 0;
+		systrace_output_rtime_state(&gdsqs[tmp], sinfo.rtime[tmp]);
+		tmp++;
+		if (tmp >= ctx.upper) {
+			atomic64_inc(&pcp_dsq_round);
+			hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n",
+							atomic64_read(&pcp_dsq_round));
+			tmp = ctx.lower;
+		}
+		spin_unlock(&sinfo.lock);
+	} while (tmp != cidx);
+
+	return consume_pcp_dsq(rq, rf, true);
+}
+
+static bool consume_hmbird_global_dsq(struct rq *rq, struct rq_flags *rf)
+{
+	enum cpu_type type = cpu_cluster(cpu_of(rq));
+	bool period_allowed = !get_hmbird_rq(rq)->period_disallow;
+	bool non_period_allowed = !get_hmbird_rq(rq)->nonperiod_disallow;
+
+	switch (type) {
+	case EX_FREE:
+		if (consume_period_dsq(rq, rf))
+			return 1;
+		if (consume_timeout_dsq(rq, rf, BIG))
+			return 1;
+		if (consume_ux_dsq(rq, rf))
+			return 1;
+		if (consume_non_period_dsq(rq, rf, BIG))
+			return 1;
+		if (is_little_need_rescue()) {
+			if (consume_timeout_dsq(rq, rf, LITTLE))
+				return 1;
+			if (consume_ux_dsq(rq, rf))
+				return 1;
+			if (consume_non_period_dsq(rq, rf, LITTLE))
+				return 1;
+		}
+		break;
+	case EXCLUSIVE:
+		if (!is_iso_par_free()) {
+			if (consume_pcp_dsq(rq, rf, true))
+				return 1;
+			break;
+		}
+		/* Only non-period task can run on isolate cpus */
+		if (!READ_ONCE(iso_free_rescue)) {
+			if (is_big_need_rescue()) {
+				if (consume_timeout_dsq(rq, rf, BIG))
+					return 1;
+				if (consume_non_period_dsq(rq, rf, BIG))
+					return 1;
+			}
+			if (is_little_need_rescue()) {
+				if (consume_timeout_dsq(rq, rf, LITTLE))
+					return 1;
+				if (consume_non_period_dsq(rq, rf, LITTLE))
+					return 1;
+			}
+		} else {
+			/* Free run, can run on any task. */
+			if (consume_period_dsq(rq, rf))
+				return 1;
+			if (consume_timeout_dsq(rq, rf, BIG))
+				return 1;
+			if (consume_timeout_dsq(rq, rf, LITTLE))
+				return 1;
+			if (consume_ux_dsq(rq, rf))
+				return 1;
+			if (consume_non_period_dsq(rq, rf, BIG))
+				return 1;
+			if (consume_non_period_dsq(rq, rf, LITTLE))
+				return 1;
+		}
+		if (consume_pcp_dsq(rq, rf, true))
+			return 1;
+		break;
+	case PARTIAL:
+		if (!is_partial_enabled()) {
+			if (consume_pcp_dsq(rq, rf, true))
+				return 1;
+			return 0;
+		}
+		if (is_big_need_rescue()) {
+			if (period_allowed && consume_period_dsq(rq, rf))
+				return 1;
+			if (non_period_allowed && consume_timeout_dsq(rq, rf, BIG))
+				return 1;
+			if (period_allowed && consume_ux_dsq(rq, rf))
+				return 1;
+			if (non_period_allowed && consume_non_period_dsq(rq, rf, BIG))
+				return 1;
+		}
+		if (is_little_need_rescue()) {
+			if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0))
+				return 1;
+			if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2))
+				return 1;
+			if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3))
+				return 1;
+			if (non_period_allowed && consume_timeout_dsq(rq, rf, LITTLE))
+				return 1;
+			if (consume_ux_dsq(rq, rf))
+				return 1;
+			if (non_period_allowed && consume_non_period_dsq(rq, rf, LITTLE))
+				return 1;
+		}
+		if (consume_pcp_dsq(rq, rf, true))
+			return 1;
+		break;
+
+	case BIG:
+		if (period_allowed && consume_period_dsq(rq, rf))
+			return 1;
+		if (non_period_allowed && consume_timeout_dsq(rq, rf, type))
+			return 1;
+		if (period_allowed && consume_ux_dsq(rq, rf))
+			return 1;
+		if (non_period_allowed && consume_non_period_dsq(rq, rf, type))
+			return 1;
+		if (is_iso_par_free() || (is_little_need_rescue() &&
+				!cluster_need_rescue(iso_masks.big, parctrl_high_ratio))) {
+			if (consume_timeout_dsq(rq, rf, LITTLE))
+				return 1;
+			if (consume_non_period_dsq(rq, rf, LITTLE)) {
+				hmbird_internal_systrace("C|9999|b_rescue_l|%d\n", b_rescue_l++);
+				return 1;
+			}
+		}
+		break;
+
+	case LITTLE:
+		if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL0))
+			return 1;
+		if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL2))
+			return 1;
+		if (consume_target_dsq(rq, rf, SCHED_PROP_DEADLINE_LEVEL3))
+			return 1;
+		if (non_period_allowed && consume_timeout_dsq(rq, rf, type))
+			return 1;
+		if (consume_ux_dsq(rq, rf))
+			return 1;
+		if (non_period_allowed && consume_non_period_dsq(rq, rf, type))
+			return 1;
+
+		if (is_iso_par_free() || (is_big_need_rescue() &&
+			!cluster_need_rescue(iso_masks.little, parctrl_high_ratio_l))) {
+			if (consume_timeout_dsq(rq, rf, BIG))
+				return 1;
+			if (consume_non_period_dsq(rq, rf, BIG)) {
+				hmbird_internal_systrace("C|9999|l_rescue_b|%d\n", l_rescue_b++);
+				return 1;
+			}
+		}
+		break;
+
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int consume_dispatch_global(struct rq *rq, struct rq_flags *rf)
+{
+	return consume_hmbird_global_dsq(rq, rf);
+}
+
+
+static void update_runningtime(struct rq *rq, struct task_struct *p, unsigned long exec_time)
+{
+	int idx;
+
+	/* which dsq belongs to while task enqueue, task will consume its running time. */
+	idx = get_hmbird_ts(p)->gdsq_idx;
+	/* Only non-period dsq share running time between each other. */
+	if (idx < NON_PERIOD_START || idx >= max_hmbird_dsq_internal_id)
+		return;
+
+	if (idx >= MAX_GLOBAL_DSQS) {
+		per_cpu(pcp_info, cpu_of(rq)).rtime += exec_time;
+		systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu_of(rq)),
+					per_cpu(pcp_info, cpu_of(rq)).rtime);
+	} else {
+		spin_lock(&sinfo.lock);
+		sinfo.rtime[idx] += exec_time;
+		spin_unlock(&sinfo.lock);
+		systrace_output_rtime_state(&gdsqs[idx], sinfo.rtime[idx]);
+	}
+}
+
+static void update_dsq_idx(struct rq *rq, struct task_struct *p, enum cpu_type type)
+{
+	int cidx;
+	struct cluster_ctx ctx;
+	int cpu = cpu_of(rq);
+
+	if (gen_cluster_ctx(&ctx, type))
+		return;
+
+	spin_lock(&sinfo.lock);
+	cidx = sinfo.curr_idx[ctx.tidx];
+	if (cidx < ctx.lower || cidx >= ctx.upper) {
+		sinfo.curr_idx[ctx.tidx] = ctx.lower;
+		hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]);
+		slim_stats_record(ERR_IDX, ctx.tidx, 0, 0);
+		cidx = sinfo.curr_idx[ctx.tidx];
+	}
+
+	while (1) {
+		if (per_cpu(pcp_info, cpu).pcp_round) {
+			if (per_cpu(pcp_info, cpu).rtime >= pcp_dsq_quota) {
+				hmbird_info_trace("cpu[%d] pcp_dsq_round is full, rtime = %d\n",
+								cpu, per_cpu(pcp_info, cpu).rtime);
+				per_cpu(pcp_info, cpu).rtime = 0;
+				per_cpu(pcp_info, cpu).pcp_round = false;
+				hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, false);
+				systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu),
+						per_cpu(pcp_info, cpu_of(rq)).rtime);
+			}
+		}
+		if (sinfo.rtime[cidx] < dsq_quota[cidx])
+			break;
+
+		/* clear current dsq rtime */
+		sinfo.rtime[cidx] = 0;
+		systrace_output_rtime_state(&gdsqs[cidx], sinfo.rtime[cidx]);
+
+		sinfo.curr_idx[ctx.tidx]++;
+		hmbird_info_systrace("C|9999|cidx_%d|%d\n", ctx.tidx, sinfo.curr_idx[ctx.tidx]);
+		if (sinfo.curr_idx[ctx.tidx] >= ctx.upper) {
+			atomic64_inc(&pcp_dsq_round);
+			hmbird_info_systrace("C|9999|pcp_dsq_round|%lld\n",
+								atomic64_read(&pcp_dsq_round));
+			sinfo.curr_idx[ctx.tidx] = ctx.lower;
+			hmbird_info_systrace("C|9999|cidx_%d|%d\n",
+								ctx.tidx, sinfo.curr_idx[ctx.tidx]);
+		}
+		cidx = sinfo.curr_idx[ctx.tidx];
+		slim_stats_record(SWITCH_IDX, 1, 0, 0);
+	}
+	spin_unlock(&sinfo.lock);
+}
+
+
+static void update_dispatch_dsq_info(struct rq *rq, struct task_struct *p)
+{
+	enum cpu_type type;
+
+	if (!rq || !p)
+		return;
+
+	type = cpu_cluster(cpu_of(rq));
+	switch (type) {
+	case PARTIAL:
+		return;
+	case EXCLUSIVE:
+		return;
+	case EX_FREE:
+		return;
+	default:
+		break;
+	}
+	update_dsq_idx(rq, p, type);
+}
+
+
+static bool scan_dsq_timeout(struct rq *rq, struct hmbird_dispatch_q *dsq, u64 deadline)
+{
+	struct hmbird_entity *entity;
+	int dsq_id;
+
+	raw_spin_lock(&dsq->lock);
+	if (list_empty(&dsq->fifo) || dsq->is_timeout) {
+		raw_spin_unlock(&dsq->lock);
+		return false;
+	}
+
+	entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo);
+	if (!entity) {
+		hmbird_deferred_err(SCAN_ENTITY_NULL,
+				"<error> : entity is NULL, dsq->id = %llu\n", dsq->id);
+		raw_spin_unlock(&dsq->lock);
+		return false;
+	}
+
+	if (time_before_eq(jiffies, entity->runnable_at + msecs_to_jiffies(deadline))) {
+		raw_spin_unlock(&dsq->lock);
+		return false;
+	}
+
+	dsq->is_timeout = true;
+	dsq_id = dsq_id_to_internal(dsq);
+	hmbird_info_trace("dsq[%d] has timeout task-%s, jiffies = %lu, runnable at = %lu\n",
+			dsq_id, entity->task->comm, jiffies, entity->runnable_at);
+	hmbird_info_systrace("C|9999|dsq_%d_timeout|%d\n", dsq_id, 1);
+	raw_spin_unlock(&dsq->lock);
+
+	return true;
+}
+
+void scan_timeout(struct rq *rq)
+{
+	int i;
+	int cpu = cpu_of(rq);
+	struct hmbird_dispatch_q *dsq;
+	static u64 last_scan_at;
+	static DEFINE_PER_CPU(u64, pcp_last_scan_at);
+
+	if (time_before_eq(jiffies, (unsigned long)per_cpu(pcp_last_scan_at, cpu)))
+		return;
+	per_cpu(pcp_last_scan_at, cpu) = jiffies;
+
+	dsq = &per_cpu(pcp_ldsq, cpu);
+	scan_dsq_timeout(rq, dsq, pcp_dsq_deadline);
+
+	if (time_before_eq(jiffies, (unsigned long)last_scan_at))
+		return;
+	last_scan_at = jiffies;
+
+	for (i = NON_PERIOD_START; i < NON_PERIOD_END; i++) {
+		dsq = &gdsqs[i];
+		scan_dsq_timeout(rq, dsq, HMBIRD_BPF_DSQS_DEADLINE[i]);
+	}
+}
+
+/*******************************Initialize***********************************/
+
+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id);
+static void init_dsq_at_boot(void)
+{
+	int i, cpu;
+
+	for (i = 0; i < MAX_GLOBAL_DSQS; i++) {
+		init_dsq(&gdsqs[i], (u64)HMBIRD_DSQ_FLAG_BUILTIN |
+				(GDSQS_ID_BASE + i));
+	}
+	for_each_possible_cpu(cpu)
+		init_dsq(&per_cpu(pcp_ldsq, cpu), (u64)HMBIRD_DSQ_FLAG_BUILTIN |
+				(GDSQS_ID_BASE + i + cpu));
+
+	max_hmbird_dsq_internal_id = GDSQS_ID_BASE + i + cpu;
+	spin_lock_init(&sinfo.lock);
+}
+
+static inline void update_cgroup_ids_table(u64 ids, int hmbird_cgroup_deadline_idx)
+{
+	if (ids < 0 || ids >= NUMS_CGROUP_KINDS) {
+		pr_err("update_cgroup_ids_tab idx err!\n");
+		return;
+	}
+	cgroup_ids_table[ids] = hmbird_cgroup_deadline_idx;
+}
+
+static int cgrp_name_to_idx(struct cgroup *cgrp)
+{
+	int idx;
+
+	if (!cgrp)
+		return -1;
+
+	if (!strcmp(cgrp->kn->name, "display")
+			|| !strcmp(cgrp->kn->name, "multimedia"))
+		idx = 5; /* 8ms */
+	else if (!strcmp(cgrp->kn->name, "top-app")
+			|| !strcmp(cgrp->kn->name, "ss-top"))
+		idx = 6; /* 16ms */
+	else if (!strcmp(cgrp->kn->name, "ssfg")
+			|| !strcmp(cgrp->kn->name, "foreground"))
+		idx = 7; /* 32ms */
+	else if (!strcmp(cgrp->kn->name, "bg")
+			|| !strcmp(cgrp->kn->name, "log")
+			|| !strcmp(cgrp->kn->name, "dex2oat")
+			|| !strcmp(cgrp->kn->name, "background"))
+		idx = 9; /* 128ms */
+	else
+		idx = DEFAULT_CGROUP_DL_IDX; /* 64ms */
+
+	return idx;
+}
+
+static void init_root_tg(struct cgroup  *cgrp, struct task_group *tg)
+{
+	if (!cgrp || !tg || !(cgrp->kn))
+		return;
+	update_cgroup_ids_table(cgrp->kn->id, DEFAULT_CGROUP_DL_IDX);
+}
+
+static void init_level1_tg(struct cgroup *cgrp, struct task_group *tg)
+{
+	if (!cgrp || !tg || !(cgrp->kn))
+		return;
+
+	if (cgrp->kn->id < 0 || cgrp->kn->id >= NUMS_CGROUP_KINDS) {
+		pr_err("%s idx err!\n", __func__);
+		return;
+	}
+
+	if (cgroup_ids_table[cgrp->kn->id] == -1)
+		update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(cgrp));
+}
+
+static void init_child_tg(struct cgroup *cgrp, struct task_group *tg)
+{
+	struct cgroup *l1cgrp;
+
+	if (!cgrp || !tg || !(cgrp->kn))
+		return;
+
+	l1cgrp = cgroup_ancestor_l1(cgrp);
+	if (l1cgrp)
+		update_cgroup_ids_table(cgrp->kn->id, cgrp_name_to_idx(l1cgrp));
+	cgroup_put(l1cgrp);
+}
+
+static void cgrp_dsq_idx_init(struct cgroup *cgrp, struct task_group *tg)
+{
+	switch (cgrp->level) {
+	case 0:
+		init_root_tg(cgrp, tg);
+		break;
+	case 1:
+		init_level1_tg(cgrp, tg);
+		break;
+	default:
+		init_child_tg(cgrp, tg);
+		break;
+	}
+}
+
+/**************************************************************************/
+
+struct hmbird_task_iter {
+	struct hmbird_entity		cursor;
+	struct task_struct		*locked;
+	struct rq			*rq;
+	struct rq_flags			rf;
+};
+
+/**
+ * hmbird_task_iter_init - Initialize a task iterator
+ * @iter: iterator to init
+ *
+ * Initialize @iter. Must be called with hmbird_tasks_lock held. Once initialized,
+ * @iter must eventually be exited with hmbird_task_iter_exit().
+ *
+ * hmbird_tasks_lock may be released between this and the first next() call or
+ * between any two next() calls. If hmbird_tasks_lock is released between two
+ * next() calls, the caller is responsible for ensuring that the task being
+ * iterated remains accessible either through RCU read lock or obtaining a
+ * reference count.
+ *
+ * All tasks which existed when the iteration started are guaranteed to be
+ * visited as long as they still exist.
+ */
+static void hmbird_task_iter_init(struct hmbird_task_iter *iter)
+{
+	lockdep_assert_held(&hmbird_tasks_lock);
+
+	iter->cursor = (struct hmbird_entity){ .flags = HMBIRD_TASK_CURSOR };
+	list_add(&iter->cursor.tasks_node, &hmbird_tasks);
+	iter->locked = NULL;
+}
+
+/**
+ * hmbird_task_iter_exit - Exit a task iterator
+ * @iter: iterator to exit
+ *
+ * Exit a previously initialized @iter. Must be called with hmbird_tasks_lock held.
+ * If the iterator holds a task's rq lock, that rq lock is released. See
+ * hmbird_task_iter_init() for details.
+ */
+static void hmbird_task_iter_exit(struct hmbird_task_iter *iter)
+{
+	struct list_head *cursor = &iter->cursor.tasks_node;
+
+	lockdep_assert_held(&hmbird_tasks_lock);
+
+	if (iter->locked) {
+		task_rq_unlock(iter->rq, iter->locked, &iter->rf);
+		iter->locked = NULL;
+	}
+
+	if (list_empty(cursor))
+		return;
+
+	list_del_init(cursor);
+}
+
+/**
+ * hmbird_task_iter_next - Next task
+ * @iter: iterator to walk
+ *
+ * Visit the next task. See hmbird_task_iter_init() for details.
+ */
+static struct task_struct *hmbird_task_iter_next(struct hmbird_task_iter *iter)
+{
+	struct list_head *cursor = &iter->cursor.tasks_node;
+	struct hmbird_entity *pos;
+
+	lockdep_assert_held(&hmbird_tasks_lock);
+
+	list_for_each_entry(pos, cursor, tasks_node) {
+		if (&pos->tasks_node == &hmbird_tasks)
+			return NULL;
+		if (!(pos->flags & HMBIRD_TASK_CURSOR)) {
+			list_move(cursor, &pos->tasks_node);
+			return pos->task;
+		}
+	}
+
+	/* can't happen, should always terminate at hmbird_tasks above */
+	hmbird_deferred_err(ITER_RET_NULL, "<error> : unreachable path in scx_task_iter_next\n");
+	return NULL;
+}
+
+/**
+ * hmbird_task_iter_next_filtered - Next non-idle task
+ * @iter: iterator to walk
+ *
+ * Visit the next non-idle task. See hmbird_task_iter_init() for details.
+ */
+static struct task_struct *
+hmbird_task_iter_next_filtered(struct hmbird_task_iter *iter)
+{
+	struct task_struct *p;
+
+	while ((p = hmbird_task_iter_next(iter))) {
+		if (!is_idle_task(p))
+			return p;
+	}
+	return NULL;
+}
+
+/**
+ * hmbird_task_iter_next_filtered_locked - Next non-idle task with its rq locked
+ * @iter: iterator to walk
+ *
+ * Visit the next non-idle task with its rq lock held. See hmbird_task_iter_init()
+ * for details.
+ */
+static struct task_struct *
+hmbird_task_iter_next_filtered_locked(struct hmbird_task_iter *iter)
+{
+	struct task_struct *p;
+
+	if (iter->locked) {
+		task_rq_unlock(iter->rq, iter->locked, &iter->rf);
+		iter->locked = NULL;
+	}
+
+	p = hmbird_task_iter_next_filtered(iter);
+	if (!p)
+		return NULL;
+
+	iter->rq = task_rq_lock(p, &iter->rf);
+	iter->locked = p;
+	return p;
+}
+
+static enum hmbird_ops_enable_state hmbird_ops_enable_state(void)
+{
+	return atomic_read(&hmbird_ops_enable_state_var);
+}
+
+static enum hmbird_ops_enable_state
+hmbird_ops_set_enable_state(enum hmbird_ops_enable_state to)
+{
+	return atomic_xchg(&hmbird_ops_enable_state_var, to);
+}
+
+static bool hmbird_ops_tryset_enable_state(enum hmbird_ops_enable_state to,
+					enum hmbird_ops_enable_state from)
+{
+	int from_v = from;
+
+	return atomic_try_cmpxchg(&hmbird_ops_enable_state_var, &from_v, to);
+}
+
+static bool hmbird_ops_disabling(void)
+{
+	return false;
+}
+
+/**
+ * wait_ops_state - Busy-wait the specified ops state to end
+ * @p: target task
+ * @opss: state to wait the end of
+ *
+ * Busy-wait for @p to transition out of @opss. This can only be used when the
+ * state part of @opss is %HMBIRD_QUEUEING or %HMBIRD_DISPATCHING. This function also
+ * has load_acquire semantics to ensure that the caller can see the updates made
+ * in the enqueueing and dispatching paths.
+ */
+static void wait_ops_state(struct task_struct *p, u64 opss)
+{
+	do {
+		cpu_relax();
+	} while (atomic64_read_acquire(&(get_hmbird_ts(p)->ops_state)) == opss);
+}
+
+
+static void update_curr_hmbird(struct rq *rq)
+{
+	struct task_struct *curr = rq->curr;
+	u64 now = rq_clock_task(rq);
+	u64 delta_exec;
+
+	if (time_before_eq64(now, curr->se.exec_start))
+		return;
+
+	delta_exec = now - curr->se.exec_start;
+	curr->se.exec_start = now;
+	update_runningtime(rq, curr, delta_exec);
+	curr->se.sum_exec_runtime += delta_exec;
+	account_group_exec_runtime(curr, delta_exec);
+	cgroup_account_cputime(curr, delta_exec);
+
+	if (get_hmbird_ts(curr)->slice != HMBIRD_SLICE_INF)
+		get_hmbird_ts(curr)->slice -= min(get_hmbird_ts(curr)->slice, delta_exec);
+
+	trace_sched_stat_runtime(curr, delta_exec, 0);
+}
+
+static bool hmbird_dsq_priq_less(struct rb_node *node_a,
+					const struct rb_node *node_b)
+{
+	const struct hmbird_entity *a =
+		container_of(node_a, struct hmbird_entity, dsq_node.priq);
+	const struct hmbird_entity *b =
+		container_of(node_b, struct hmbird_entity, dsq_node.priq);
+
+	return time_before64(a->dsq_vtime, b->dsq_vtime);
+}
+
+static void dispatch_enqueue(struct hmbird_dispatch_q *dsq, struct task_struct *p,
+							u64 enq_flags)
+{
+	bool is_local = dsq->id == HMBIRD_DSQ_LOCAL;
+	unsigned long flags;
+
+	hmbird_cond_deferred_err(ENQ_EXIST1, get_hmbird_ts(p)->dsq ||
+				!list_empty(&get_hmbird_ts(p)->dsq_node.fifo),
+				"task = %s, dsq->id = %llu\n", p->comm, dsq->id);
+	hmbird_cond_deferred_err(ENQ_EXIST2,
+				(get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) ||
+				!RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq),
+				"task = %s\n", p->comm);
+
+	if (!is_local) {
+		raw_spin_lock_irqsave(&dsq->lock, flags);
+		if (unlikely(dsq->id == HMBIRD_DSQ_INVALID)) {
+			WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_DSQ);
+			hmbird_ops_error("<hmbird_sched>: %s\n",
+					"attempting to dispatch to a destroyed dsq");
+			/* fall back to the global dsq */
+			raw_spin_unlock_irqrestore(&dsq->lock, flags);
+			dsq = &hmbird_dsq_global;
+			raw_spin_lock_irqsave(&dsq->lock, flags);
+		}
+	}
+
+	if (enq_flags & HMBIRD_ENQ_DSQ_PRIQ) {
+		get_hmbird_ts(p)->dsq_flags |= HMBIRD_TASK_DSQ_ON_PRIQ;
+		rb_add_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq,
+					hmbird_dsq_priq_less);
+	} else {
+		if (enq_flags & (HMBIRD_ENQ_HEAD | HMBIRD_ENQ_PREEMPT))
+			list_add(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo);
+		else
+			list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &dsq->fifo);
+	}
+	dsq->nr++;
+	get_hmbird_ts(p)->dsq = dsq;
+
+	/*
+	 * We're transitioning out of QUEUEING or DISPATCHING. store_release to
+	 * match waiters' load_acquire.
+	 */
+	if (enq_flags & HMBIRD_ENQ_CLEAR_OPSS)
+		atomic64_set_release(&get_hmbird_ts(p)->ops_state, HMBIRD_OPSS_NONE);
+
+	if (is_local) {
+		struct hmbird_rq *hmbird = container_of(dsq, struct hmbird_rq, local_dsq);
+		struct rq *rq = hmbird->rq;
+		bool preempt = false;
+
+		if ((enq_flags & HMBIRD_ENQ_PREEMPT) && p != rq->curr &&
+			rq->curr->sched_class == &hmbird_sched_class) {
+			get_hmbird_ts(rq->curr)->slice = 0;
+			preempt = true;
+		}
+
+		if (preempt || sched_class_above(&hmbird_sched_class,
+						rq->curr->sched_class))
+			resched_curr(rq);
+	} else {
+		raw_spin_unlock_irqrestore(&dsq->lock, flags);
+	}
+}
+
+static void task_unlink_from_dsq(struct task_struct *p,
+				struct hmbird_dispatch_q *dsq)
+{
+	if (get_hmbird_ts(p)->dsq_flags & HMBIRD_TASK_DSQ_ON_PRIQ) {
+		rb_erase_cached(&get_hmbird_ts(p)->dsq_node.priq, &dsq->priq);
+		RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq);
+		get_hmbird_ts(p)->dsq_flags &= ~HMBIRD_TASK_DSQ_ON_PRIQ;
+	} else {
+		list_del_init(&get_hmbird_ts(p)->dsq_node.fifo);
+	}
+}
+
+static bool task_linked_on_dsq(struct task_struct *p)
+{
+	return !list_empty(&get_hmbird_ts(p)->dsq_node.fifo) ||
+		!RB_EMPTY_NODE(&get_hmbird_ts(p)->dsq_node.priq);
+}
+
+static void dispatch_dequeue(struct hmbird_rq *hmbird_rq, struct task_struct *p)
+{
+	unsigned long flags;
+	struct hmbird_dispatch_q *dsq = get_hmbird_ts(p)->dsq;
+	bool is_local = dsq == &hmbird_rq->local_dsq;
+
+	if (!dsq) {
+		hmbird_cond_deferred_err(TASK_LINKED1,
+						task_linked_on_dsq(p), "task = %s\n", p->comm);
+		/*
+		 * When dispatching directly from the BPF scheduler to a local
+		 * DSQ, the task isn't associated with any DSQ but
+		 * @get_hmbird_ts(p)->holding_cpu may be set under the protection of
+		 * %HMBIRD_OPSS_DISPATCHING.
+		 */
+		if (get_hmbird_ts(p)->holding_cpu >= 0)
+			get_hmbird_ts(p)->holding_cpu = -1;
+		return;
+	}
+
+	if (!is_local)
+		raw_spin_lock_irqsave(&dsq->lock, flags);
+
+	/*
+	 * Now that we hold @dsq->lock, @p->holding_cpu and @get_hmbird_ts(p)->dsq_node
+	 * can't change underneath us.
+	 */
+	if (get_hmbird_ts(p)->holding_cpu < 0) {
+		/* @p must still be on @dsq, dequeue */
+		hmbird_cond_deferred_err(TASK_UNLINKED,
+						!task_linked_on_dsq(p), "task = %s\n", p->comm);
+		task_unlink_from_dsq(p, dsq);
+		dsq->nr--;
+	} else {
+		/*
+		 * We're racing against dispatch_to_local_dsq() which already
+		 * removed @p from @dsq and set @get_hmbird_ts(p)->holding_cpu. Clear the
+		 * holding_cpu which tells dispatch_to_local_dsq() that it lost
+		 * the race.
+		 */
+		hmbird_cond_deferred_err(TASK_LINKED2,
+						task_linked_on_dsq(p), "task = %s\n", p->comm);
+		get_hmbird_ts(p)->holding_cpu = -1;
+	}
+	get_hmbird_ts(p)->dsq = NULL;
+
+	if (!is_local)
+		raw_spin_unlock_irqrestore(&dsq->lock, flags);
+}
+
+
+static bool test_rq_online(struct rq *rq)
+{
+#ifdef CONFIG_SMP
+	return rq->online;
+#else
+	return true;
+#endif
+}
+
+static void refill_task_slice(struct task_struct *p)
+{
+	if (is_isolate_task(p))
+		get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO;
+	else if (is_pipeline_task(p))
+		get_hmbird_ts(p)->slice = HMBIRD_SLICE_ISO / 2;
+	else
+		get_hmbird_ts(p)->slice = HMBIRD_SLICE_DFL;
+}
+
+static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
+				int sticky_cpu)
+{
+	struct hmbird_dispatch_q *d;
+	s32 cpu = -1;
+
+	hmbird_cond_deferred_err(TASK_UNQUED, !test_bit(ffs(HMBIRD_TASK_QUEUED),
+				(unsigned long *)&get_hmbird_ts(p)->flags),
+				"task = %s\n", p->comm);
+
+	if (is_pcp_rt(p)) {
+		/* Enqueue percpu rt task to local directly. */
+		/* Or cause a bug when disable dispatch. */
+		if (cpumask_test_cpu(cpu_of(rq), p->cpus_ptr))
+			enq_flags |= HMBIRD_ENQ_LOCAL;
+	}
+
+	cpu = get_hmbird_ts(p)->critical_affinity_cpu;
+	if (cpu >= 0) {
+		set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags);
+	}
+
+	if (test_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags)) {
+		enq_flags |= HMBIRD_ENQ_LOCAL;
+		clear_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags);
+	}
+	/* rq migration */
+	if (sticky_cpu == cpu_of(rq))
+		goto local_norefill;
+	/*
+	 * If !rq->online, we already told the BPF scheduler that the CPU is
+	 * offline. We're just trying to on/offline the CPU. Don't bother the
+	 * BPF scheduler.
+	 */
+	if (unlikely(!test_rq_online(rq)))
+		goto local;
+
+	/* see %HMBIRD_OPS_ENQ_LAST */
+	if (enq_flags & HMBIRD_ENQ_LAST)
+		goto local;
+
+	if (enq_flags & HMBIRD_ENQ_LOCAL)
+		goto local;
+	else
+		goto global;
+local:
+	/*
+	 * For task-ordering, slice refill must be treated as implying the end
+	 * of the current slice. Otherwise, the longer @p stays on the CPU, the
+	 * higher priority it becomes from hmbird_prio_less()'s POV.
+	 */
+	refill_task_slice(p);
+local_norefill:
+	dispatch_enqueue(&get_hmbird_rq(rq)->local_dsq, p, enq_flags);
+	slim_stats_record(PCP_ENQL_CNT, 0, 0, cpu_of(rq));
+	return;
+
+global:
+	d = find_dsq_from_task(p);
+	if (d) {
+		refill_task_slice(p);
+		dispatch_enqueue(d, p, enq_flags);
+		return;
+	}
+	slim_stats_record(GLOBAL_STAT, 0, 0, 0);
+	refill_task_slice(p);
+	dispatch_enqueue(&hmbird_dsq_global, p, enq_flags);
+}
+
+static bool watchdog_task_watched(const struct task_struct *p)
+{
+	return !list_empty(&get_hmbird_ts(p)->watchdog_node);
+}
+
+static void watchdog_watch_task(struct rq *rq, struct task_struct *p)
+{
+	lockdep_assert_rq_held(rq);
+	if (test_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags))
+		get_hmbird_ts(p)->runnable_at = jiffies;
+	clear_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags);
+	list_add_tail(&get_hmbird_ts(p)->watchdog_node, &get_hmbird_rq(rq)->watchdog_list);
+}
+
+static void watchdog_unwatch_task(struct task_struct *p, bool reset_timeout)
+{
+	list_del_init(&get_hmbird_ts(p)->watchdog_node);
+	if (reset_timeout)
+		set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags);
+}
+
+static void enqueue_task_hmbird(struct rq *rq, struct task_struct *p, int enq_flags)
+{
+	int sticky_cpu = get_hmbird_ts(p)->sticky_cpu;
+
+	enq_flags |= get_hmbird_rq(rq)->extra_enq_flags;
+
+	if (sticky_cpu >= 0)
+		get_hmbird_ts(p)->sticky_cpu = -1;
+
+	/*
+	 * Restoring a running task will be immediately followed by
+	 * set_next_task_hmbird() which expects the task to not be on the BPF
+	 * scheduler as tasks can only start running through local DSQs. Force
+	 * direct-dispatch into the local DSQ by setting the sticky_cpu.
+	 */
+	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p))
+		sticky_cpu = cpu_of(rq);
+
+	if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) {
+		hmbird_cond_deferred_err(TASK_UNWATCHED,
+			!watchdog_task_watched(p), "task = %s\n", p->comm);
+		return;
+	}
+
+	watchdog_watch_task(rq, p);
+	set_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags);
+	get_hmbird_rq(rq)->nr_running++;
+	add_nr_running(rq, 1);
+
+	do_enqueue_task(rq, p, enq_flags, sticky_cpu);
+}
+
+static void ops_dequeue(struct task_struct *p, u64 deq_flags)
+{
+	u64 opss;
+
+	watchdog_unwatch_task(p, false);
+
+	/* acquire ensures that we see the preceding updates on QUEUED */
+	opss = atomic64_read_acquire(&get_hmbird_ts(p)->ops_state);
+
+	switch (opss & HMBIRD_OPSS_STATE_MASK) {
+	case HMBIRD_OPSS_NONE:
+		break;
+	case HMBIRD_OPSS_QUEUEING:
+		/*
+		 * QUEUEING is started and finished while holding @p's rq lock.
+		 * As we're holding the rq lock now, we shouldn't see QUEUEING.
+		 */
+		hmbird_deferred_err(DEQ_DEQING, "<error> : unreachable path in %s\n", __func__);
+		break;
+	case HMBIRD_OPSS_QUEUED:
+		if (atomic64_try_cmpxchg(&get_hmbird_ts(p)->ops_state, &opss,
+					 HMBIRD_OPSS_NONE))
+			break;
+		fallthrough;
+	case HMBIRD_OPSS_DISPATCHING:
+		/*
+		 * If @p is being dispatched from the BPF scheduler to a DSQ,
+		 * wait for the transfer to complete so that @p doesn't get
+		 * added to its DSQ after dequeueing is complete.
+		 *
+		 * As we're waiting on DISPATCHING with the rq locked, the
+		 * dispatching side shouldn't try to lock the rq while
+		 * DISPATCHING is set. See dispatch_to_local_dsq().
+		 *
+		 * DISPATCHING shouldn't have qseq set and control can reach
+		 * here with NONE @opss from the above QUEUED case block.
+		 * Explicitly wait on %HMBIRD_OPSS_DISPATCHING instead of @opss.
+		 */
+		wait_ops_state(p, HMBIRD_OPSS_DISPATCHING);
+		hmbird_cond_deferred_err(HMBIRD_OPN,
+			atomic64_read(&get_hmbird_ts(p)->ops_state) != HMBIRD_OPSS_NONE,
+						"task = %s\n", p->comm);
+		break;
+	}
+}
+
+static void dequeue_task_hmbird(struct rq *rq, struct task_struct *p, int deq_flags)
+{
+	struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq);
+
+	if (!test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) {
+		hmbird_cond_deferred_err(TASK_WATCHED, watchdog_task_watched(p),
+						"task = %s\n", p->comm);
+		return;
+	}
+
+	ops_dequeue(p, deq_flags);
+
+	if (slim_walt_ctrl) {
+		if (task_current(rq, p))
+			hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK);
+	}
+
+	if (deq_flags & HMBIRD_DEQ_SLEEP)
+		set_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP), (unsigned long *)&get_hmbird_ts(p)->flags);
+	else
+		clear_bit(ffs(HMBIRD_TASK_DEQD_FOR_SLEEP),
+			(unsigned long *)&get_hmbird_ts(p)->flags);
+
+	clear_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags);
+	hmbird_cond_deferred_err(RQ_NO_RUNNING, !hmbird_rq->nr_running, "task = %s\n", p->comm);
+	hmbird_rq->nr_running--;
+	sub_nr_running(rq, 1);
+	dispatch_dequeue(hmbird_rq, p);
+}
+
+static void yield_task_hmbird(struct rq *rq)
+{
+	struct task_struct *p = rq->curr;
+
+	get_hmbird_ts(p)->slice = 0;
+}
+
+static bool yield_to_task_hmbird(struct rq *rq, struct task_struct *to)
+{
+	return false;
+}
+
+#ifdef CONFIG_SMP
+/**
+ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ
+ * @rq: rq to move the task into, currently locked
+ * @p: task to move
+ * @enq_flags: %HMBIRD_ENQ_*
+ *
+ * Move @p which is currently on a different rq to @rq's local DSQ. The caller
+ * must:
+ *
+ * 1. Start with exclusive access to @p either through its DSQ lock or
+ *    %HMBIRD_OPSS_DISPATCHING flag.
+ *
+ * 2. Set @get_hmbird_ts(p)->holding_cpu to raw_smp_processor_id().
+ *
+ * 3. Remember task_rq(@p). Release the exclusive access so that we don't
+ *    deadlock with dequeue.
+ *
+ * 4. Lock @rq and the task_rq from #3.
+ *
+ * 5. Call this function.
+ *
+ * Returns %true if @p was successfully moved. %false after racing dequeue and
+ * losing.
+ */
+static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p,
+					u64 enq_flags)
+{
+	struct rq *task_rq;
+
+	lockdep_assert_rq_held(rq);
+
+	/*
+	 * If dequeue got to @p while we were trying to lock both rq's, it'd
+	 * have cleared @get_hmbird_ts(p)->holding_cpu to -1. While other cpus may have
+	 * updated it to different values afterwards, as this operation can't be
+	 * preempted or recurse, @get_hmbird_ts(p)->holding_cpu can never become
+	 * raw_smp_processor_id() again before we're done. Thus, we can tell
+	 * whether we lost to dequeue by testing whether @get_hmbird_ts(p)->holding_cpu is
+	 * still raw_smp_processor_id().
+	 *
+	 * See dispatch_dequeue() for the counterpart.
+	 */
+	if (unlikely(get_hmbird_ts(p)->holding_cpu != raw_smp_processor_id()))
+		return false;
+
+	/* @p->rq couldn't have changed if we're still the holding cpu */
+	task_rq = task_rq(p);
+	lockdep_assert_rq_held(task_rq);
+	deactivate_task(task_rq, p, 0);
+	set_task_cpu(p, cpu_of(rq));
+	get_hmbird_ts(p)->sticky_cpu = cpu_of(rq);
+
+	/*
+	 * We want to pass hmbird-specific enq_flags but activate_task() will
+	 * truncate the upper 32 bit. As we own @rq, we can pass them through
+	 * @get_hmbird_rq(rq)->extra_enq_flags instead.
+	 */
+	hmbird_cond_deferred_err(EXTRA_FLAGS, get_hmbird_rq(rq)->extra_enq_flags,
+					"task = %s\n", p->comm);
+	get_hmbird_rq(rq)->extra_enq_flags = enq_flags;
+	activate_task(rq, p, 0);
+	get_hmbird_rq(rq)->extra_enq_flags = 0;
+
+	return true;
+}
+
+#endif	/* CONFIG_SMP */
+
+static int task_fits_cpu_hmbird(struct task_struct *p, int cpu)
+{
+	int fitable = 1;
+
+	return fitable;
+}
+
+static int check_misfit_task_on_little(struct task_struct *p, struct rq *rq,
+						struct hmbird_dispatch_q *dsq)
+{
+	bool dsq_misfit;
+	int cpu = cpu_of(rq);
+	u64 task_util = 0;
+	struct cluster_ctx ctx;
+	int dsq_int = dsq_id_to_internal(dsq);
+
+	if (!cpumask_test_cpu(cpu, iso_masks.little))
+		return false;
+
+	gen_cluster_ctx(&ctx, BIG);
+	dsq_misfit = (dsq_int >= SCHED_PROP_DEADLINE_LEVEL1 &&
+				dsq_int <= SCHED_PROP_DEADLINE_LEVEL4);
+#ifdef CLUSTER_SEPARATE
+	/* In rescue mode, little will consume big cluster's dsq.*/
+	dsq_misfit |= (dsq_int >= ctx.lower && dsq_int < ctx.upper);
+#endif
+	if (!dsq_misfit)
+		return false;
+
+	if (p) {
+		if (slim_walt_ctrl)
+			slim_get_task_util(p, &task_util);
+		else
+			task_util = get_hmbird_ts(p)->demand_scaled;
+	}
+
+	if (task_util <= misfit_ds)
+		return false;
+
+	hmbird_info_trace("<filter>:task %s can't run on cpu%d, util = %llu\n",
+							p->comm, cpu, task_util);
+	return true;
+}
+
+static bool task_can_run_on_rq(struct task_struct *p, struct rq *rq, struct hmbird_dispatch_q *dsq)
+{
+	if (!task_fits_cpu_hmbird(p, cpu_of(rq)))
+		return false;
+
+	if (check_misfit_task_on_little(p, rq, dsq))
+		return false;
+
+	return likely(test_rq_online(rq));
+}
+
+static void set_skip_num(struct hmbird_dispatch_q *dsq, int *skipn, bool add)
+{
+	int idx = dsq_id_to_internal(dsq);
+	int type = get_dsq_type(dsq);
+
+	if (type != GLOBAL_DSQ)
+		return;
+
+	if (add)
+		skipn[idx]++;
+	else
+		skipn[idx] = 0;
+}
+
+static bool skip_too_much(struct hmbird_dispatch_q *dsq)
+{
+	int idx = dsq_id_to_internal(dsq);
+	int type = get_dsq_type(dsq);
+
+	if (type != GLOBAL_DSQ)
+		return false;
+
+	if (skip_num[idx] > 3) {
+		skip_num[idx] = 0;
+		return true;
+	}
+
+	return false;
+}
+
+bool consume_dispatch_q(struct rq *rq, struct rq_flags *rf,
+					struct hmbird_dispatch_q *dsq)
+{
+	struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq);
+	struct hmbird_entity *entity;
+	struct task_struct *p;
+	struct rb_node *rb_node;
+	struct rq *task_rq;
+	unsigned long flags;
+	bool moved = false;
+	struct task_struct *may_fit = NULL;
+	int skip = 0;
+
+retry:
+	if (list_empty(&dsq->fifo) && !rb_first_cached(&dsq->priq))
+		return false;
+
+	raw_spin_lock_irqsave(&dsq->lock, flags);
+
+	list_for_each_entry(entity, &dsq->fifo, dsq_node.fifo) {
+		p = entity->task;
+		task_rq = task_rq(p);
+		if (!task_can_run_on_rq(p, rq, dsq))
+			continue;
+		if (rq == task_rq) {
+			set_skip_num(dsq, skip_num, (bool)may_fit);
+			goto this_rq;
+		}
+		if (skip_too_much(dsq))
+			goto remote_rq;
+		if (!may_fit)
+			may_fit = p;
+		if (++skip <= 3)
+			continue;
+		/*
+		 * If the recent 3 tasks not fit, use the first one.
+		 * and clear the skip, because the first one is consumed.
+		 */
+		set_skip_num(dsq, skip_num, false);
+		p = may_fit;
+		task_rq = task_rq(p);
+		goto remote_rq;
+	}
+	/* No more task, use the first may fit task.*/
+	if (may_fit) {
+		p = may_fit;
+		task_rq = task_rq(p);
+		goto remote_rq;
+	}
+
+	for (rb_node = rb_first_cached(&dsq->priq); rb_node; rb_node = rb_next(rb_node)) {
+		entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq);
+		p = entity->task;
+		task_rq = task_rq(p);
+		if (!task_can_run_on_rq(p, rq, dsq))
+			continue;
+		if (rq == task_rq)
+			goto this_rq;
+		goto remote_rq;
+	}
+
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+	return false;
+
+this_rq:
+	/* @dsq is locked and @p is on this rq */
+	hmbird_cond_deferred_err(HOLDING_CPU1, get_hmbird_ts(p)->holding_cpu >= 0,
+					"task = %s\n", p->comm);
+	task_unlink_from_dsq(p, dsq);
+	list_add_tail(&get_hmbird_ts(p)->dsq_node.fifo, &hmbird_rq->local_dsq.fifo);
+	dsq->nr--;
+	hmbird_rq->local_dsq.nr++;
+	get_hmbird_ts(p)->dsq = &hmbird_rq->local_dsq;
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+	slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0);
+	return true;
+
+remote_rq:
+#ifdef CONFIG_SMP
+	if (cpu_same_cluster_stat(p, rq, task_rq))
+		slim_stats_record(MOVE_RQ_CNT, 0, 0, 0);
+	else
+		slim_stats_record(MOVE_RQ_CNT, 1, 0, 0);
+	/*
+	 * @dsq is locked and @p is on a remote rq. @p is currently protected by
+	 * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab
+	 * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the
+	 * rq lock or fail, do a little dancing from our side. See
+	 * move_task_to_local_dsq().
+	 */
+	hmbird_cond_deferred_err(HOLDING_CPU2, get_hmbird_ts(p)->holding_cpu >= 0,
+					"task = %s\n", p->comm);
+	task_unlink_from_dsq(p, dsq);
+	dsq->nr--;
+	get_hmbird_ts(p)->holding_cpu = raw_smp_processor_id();
+	raw_spin_unlock_irqrestore(&dsq->lock, flags);
+
+	rq_unpin_lock(rq, rf);
+	double_lock_balance(rq, task_rq);
+	rq_repin_lock(rq, rf);
+
+	moved = move_task_to_local_dsq(rq, p, 0);
+
+	double_unlock_balance(rq, task_rq);
+#endif /* CONFIG_SMP */
+	if (likely(moved)) {
+		slim_stats_record(TOTAL_DSP_CNT, 0, 0, 0);
+		return true;
+	}
+	may_fit = NULL;
+	goto retry;
+}
+
+
+static int balance_one(struct rq *rq, struct task_struct *prev,
+				struct rq_flags *rf, bool local)
+{
+	struct hmbird_rq *hmbird_rq = get_hmbird_rq(rq);
+	bool prev_on_hmbird = prev->sched_class == &hmbird_sched_class;
+
+	if (!hmbird_rq)
+		return 1;
+
+	lockdep_assert_rq_held(rq);
+
+	if (static_branch_unlikely(&hmbird_ops_cpu_preempt) &&
+		unlikely(get_hmbird_rq(rq)->cpu_released)) {
+		/*
+		 * If the previous sched_class for the current CPU was not HMBIRD,
+		 * notify the BPF scheduler that it again has control of the
+		 * core. This callback complements ->cpu_release(), which is
+		 * emitted in hmbird_notify_pick_next_task().
+		 */
+		get_hmbird_rq(rq)->cpu_released = false;
+	}
+
+	if (prev_on_hmbird)
+		update_curr_hmbird(rq);
+	/* if there already are tasks to run, nothing to do */
+	if (hmbird_rq->local_dsq.nr)
+		return 1;
+
+	if (consume_dispatch_q(rq, rf, &hmbird_dsq_global)) {
+		slim_stats_record(GLOBAL_STAT, 1, 0, 0);
+		return 1;
+	}
+
+	if (consume_dispatch_global(rq, rf))
+		return 1;
+
+	return 0;
+}
+
+static int balance_hmbird(struct rq *rq, struct task_struct *prev,
+						struct rq_flags *rf)
+{
+	return balance_one(rq, prev, rf, true);
+}
+
+/*
+ * output task util to systrace, only for debug mode.
+ * we can not output too many logs to systrace buffer even in debug mode
+ * only output debug-info while it exceed misfit_ds.
+ */
+static void systrace_output_cpu_ds(struct rq *rq, struct task_struct *p)
+{
+	static DEFINE_PER_CPU(int, is_last_exceed);
+	int cpu = cpu_of(rq);
+	u64 util = 0;
+
+	if (likely(!debug_enabled()))
+		return;
+
+	if (!p)
+		return;
+
+	if (slim_walt_ctrl)
+		slim_get_task_util(p, &util);
+	else
+		util = get_hmbird_ts(p)->demand_scaled;
+	util = uclamp_rq_util_with(rq, util, p);
+
+	if (util >= misfit_ds) {
+		hmbird_internal_systrace("C|9999|cpu_%d_ds|%llu\n", cpu, util);
+		per_cpu(is_last_exceed, cpu) = true;
+	} else if (per_cpu(is_last_exceed, cpu) && (util < misfit_ds)) {
+		hmbird_internal_systrace("C|9999|cpu_%d_ds|%d\n", cpu, 0);
+		per_cpu(is_last_exceed, cpu) = false;
+	} else {
+	}
+}
+
+static void set_next_task_hmbird(struct rq *rq, struct task_struct *p, bool first)
+{
+	if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) {
+		/*
+		 * Core-sched might decide to execute @p before it is
+		 * dispatched. Call ops_dequeue() to notify the BPF scheduler.
+		 */
+		ops_dequeue(p, HMBIRD_DEQ_CORE_SCHED_EXEC);
+		dispatch_dequeue(get_hmbird_rq(rq), p);
+	}
+
+	p->se.exec_start = rq_clock_task(rq);
+
+	if (slim_walt_ctrl) {
+		if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags))
+			hmbird_update_task_ravg_rqclock_wrapper(p, rq, PICK_NEXT_TASK);
+	}
+
+	watchdog_unwatch_task(p, true);
+	slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), get_hmbird_ts(p)->gdsq_idx);
+	systrace_output_cpu_ds(rq, p);
+	/*
+	 * @p is getting newly scheduled or got kicked after someone updated its
+	 * slice. Refresh whether tick can be stopped. See can_stop_tick_hmbird().
+	 */
+	if ((get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF) !=
+	    (bool)(get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK)) {
+		if (get_hmbird_ts(p)->slice == HMBIRD_SLICE_INF)
+			get_hmbird_rq(rq)->flags |= HMBIRD_RQ_CAN_STOP_TICK;
+		else
+			get_hmbird_rq(rq)->flags &= ~HMBIRD_RQ_CAN_STOP_TICK;
+
+		sched_update_tick_dependency(rq);
+	}
+
+	p->se.prev_sum_exec_runtime = p->se.sum_exec_runtime;
+}
+
+static void put_prev_task_hmbird(struct rq *rq, struct task_struct *p)
+{
+	update_curr_hmbird(rq);
+
+	update_dispatch_dsq_info(rq, p);
+
+	if (slim_walt_ctrl) {
+		if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags))
+			hmbird_update_task_ravg_rqclock_wrapper(p, rq, PUT_PREV_TASK);
+	}
+
+	slim_trace_show_cpu_consume_dsq_idx(smp_processor_id(), 0);
+
+	if (test_bit(ffs(HMBIRD_TASK_QUEUED), (unsigned long *)&get_hmbird_ts(p)->flags)) {
+		watchdog_watch_task(rq, p);
+
+		if (is_pipeline_task(p)) {
+			do_enqueue_task(rq, p, HMBIRD_ENQ_LOCAL, -1);
+			return;
+		}
+
+		/*
+		 * If we're in the pick_next_task path, balance_hmbird() should
+		 * have already populated the local DSQ if there are any other
+		 * available tasks. If empty, tell ops.enqueue() that @p is the
+		 * only one available for this cpu. ops.enqueue() should put it
+		 * on the local DSQ so that the subsequent pick_next_task_hmbird()
+		 * can find the task unless it wants to trigger a separate
+		 * follow-up scheduling event.
+		 */
+		if (list_empty(&get_hmbird_rq(rq)->local_dsq.fifo))
+			do_enqueue_task(rq, p, HMBIRD_ENQ_LAST | HMBIRD_ENQ_LOCAL, -1);
+		else
+			do_enqueue_task(rq, p, 0, -1);
+	}
+}
+
+static struct task_struct *first_local_task(struct rq *rq)
+{
+	struct rb_node *rb_node;
+	struct hmbird_entity *entity;
+
+	if (!list_empty(&get_hmbird_rq(rq)->local_dsq.fifo)) {
+		entity = list_first_entry(&get_hmbird_rq(rq)->local_dsq.fifo,
+							struct hmbird_entity, dsq_node.fifo);
+		return entity->task;
+	}
+
+	rb_node = rb_first_cached(&get_hmbird_rq(rq)->local_dsq.priq);
+	if (rb_node) {
+		entity = container_of(rb_node, struct hmbird_entity, dsq_node.priq);
+		return entity->task;
+	}
+	return NULL;
+}
+
+static struct task_struct *pick_next_task_hmbird(struct rq *rq)
+{
+	struct task_struct *p;
+
+	p = first_local_task(rq);
+	if (!p)
+		return NULL;
+
+	if (unlikely(!get_hmbird_ts(p)->slice)) {
+		if (!hmbird_ops_disabling() && !hmbird_warned_zero_slice)
+			hmbird_warned_zero_slice = true;
+
+		refill_task_slice(p);
+	}
+
+	set_next_task_hmbird(rq, p, true);
+
+	return p;
+}
+
+void __hmbird_notify_pick_next_task(struct rq *rq, struct task_struct *task,
+				const struct sched_class *active)
+{
+	lockdep_assert_rq_held(rq);
+
+	/*
+	 * The callback is conceptually meant to convey that the CPU is no
+	 * longer under the control of HMBIRD. Therefore, don't invoke the
+	 * callback if the CPU is staying on HMBIRD, or going idle (in which
+	 * case the HMBIRD scheduler has actively decided not to schedule any
+	 * tasks on the CPU).
+	 */
+	if (likely(active >= &hmbird_sched_class))
+		return;
+
+	/*
+	 * At this point we know that HMBIRD was preempted by a higher priority
+	 * sched_class, so invoke the ->cpu_release() callback if we have not
+	 * done so already. We only send the callback once between HMBIRD being
+	 * preempted, and it regaining control of the CPU.
+	 *
+	 * ->cpu_release() complements ->cpu_acquire(), which is emitted the
+	 *  next time that balance_hmbird() is invoked.
+	 */
+	if (!get_hmbird_rq(rq)->cpu_released)
+		get_hmbird_rq(rq)->cpu_released = true;
+}
+
+#ifdef CONFIG_SMP
+
+static bool test_and_clear_cpu_idle(int cpu)
+{
+	if (cpumask_test_and_clear_cpu(cpu, idle_masks.cpu)) {
+		if (cpumask_empty(idle_masks.cpu))
+			hmbird_has_idle_cpus = false;
+		return true;
+	} else {
+		return false;
+	}
+}
+
+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed)
+{
+	int cpu;
+
+	do {
+		cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed);
+		if (cpu >= nr_cpu_ids)
+			return -EBUSY;
+	} while (!test_and_clear_cpu_idle(cpu));
+
+	return cpu;
+}
+
+
+static bool prev_cpu_misfit(int prev)
+{
+	if (!is_partial_enabled() && is_partial_cpu(prev))
+		return true;
+
+	return false;
+}
+
+static int heavy_rt_placement(struct task_struct *p, int prev)
+{
+	struct cpumask tmp = {.bits = {0}};
+	int cpu;
+	u64 util = 0;
+
+	if (!rt_prio(p->prio))
+		return -EFAULT;
+
+	if (slim_walt_ctrl)
+		slim_get_task_util(p, &util);
+	else
+		util = get_hmbird_ts(p)->demand_scaled;
+
+	if (util < misfit_ds)
+		return -EFAULT;
+
+	if (is_partial_enabled())
+		cpumask_or(&tmp, iso_masks.big, iso_masks.partial);
+	else
+		cpumask_copy(&tmp, iso_masks.big);
+
+	cpu = hmbird_pick_idle_cpu(&tmp);
+	if (cpu >= 0)
+		return cpu;
+
+	if (cpumask_test_cpu(prev, iso_masks.big) ||
+		(is_partial_enabled() && is_partial_cpu(prev)))
+		return prev;
+
+	return cpumask_first(&tmp);
+}
+
+static int spec_task_before_pick_idle(struct task_struct *p, int prev)
+{
+	int cpu;
+
+	cpu = heavy_rt_placement(p, prev);
+	if (cpu >= 0)
+		return cpu;
+	return -EFAULT;
+}
+
+static int cpumask_distribute_next(struct cpumask *mask, int *prev)
+{
+	int p = *prev, n;
+
+	n = find_next_bit_wrap(cpumask_bits(mask), nr_cpumask_bits, p + 1);
+	if (n < nr_cpu_ids)
+		WRITE_ONCE(*prev, n);
+	return n;
+}
+
+static int repick_fallback_cpu(void)
+{
+	/*
+	 * partial cpu follow big cluster's Scheduling policy,
+	 * simply return first bit cpu.
+	 */
+	return cpumask_distribute_next(iso_masks.big, &big_distribute_mask_prev);
+}
+
+/*
+ * Must return a valid cpu num, as this task's cpu.
+ */
+static int select_cpu_from_cluster(struct task_struct *p, int prev_cpu,
+					struct cpumask *mask, int *prev_mask)
+{
+	int cpu;
+
+	cpu = hmbird_pick_idle_cpu(mask);
+	if (cpu >= 0)
+		return cpu;
+	return cpumask_distribute_next(mask, prev_mask);
+}
+
+static bool task_only_blongs_to_cluster(struct task_struct *p, enum cpu_type type)
+{
+	int idx;
+	struct cluster_ctx ctx;
+
+	idx = find_idx_from_task(p);
+	if (idx < NON_PERIOD_START || idx >= MAX_GLOBAL_DSQS)
+		return false;
+
+	gen_cluster_ctx(&ctx, type);
+	if (idx >= ctx.lower && idx < ctx.upper)
+		return true;
+	return false;
+}
+
+static bool is_valid_cpu(int cpu)
+{
+	return (cpu >= 0) && (cpu < nr_cpu_ids);
+}
+
+static s32 hmbird_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags)
+{
+	s32 cpu = -1;
+	struct cpumask mask = {.bits = {0}};
+
+	cpu = get_hmbird_ts(p)->critical_affinity_cpu;
+	if (is_valid_cpu(cpu)) {
+		set_bit(ffs(HMBIRD_TASK_ENQ_LOCAL), (unsigned long *)&get_hmbird_ts(p)->flags);
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		return cpu;
+	}
+
+	partial_dynamic_ctrl();
+
+	if (is_critical_app_task_without_isolate(p) && !cpumask_empty(iso_masks.big)) {
+		cpu = select_cpu_from_cluster(p, prev_cpu,
+				iso_masks.big, &big_distribute_mask_prev);
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		if (is_valid_cpu(cpu))
+			return cpu;
+	}
+
+	if (p->nr_cpus_allowed == 1) {
+		cpu = cpumask_any(p->cpus_ptr);
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		return cpu;
+	}
+
+	/* For non-period global dsq, not contain pcp task. */
+	if (task_only_blongs_to_cluster(p, LITTLE)) {
+		cpumask_copy(&mask, iso_masks.little);
+		if (unlikely(l_need_rescue))
+			cpumask_or(&mask, iso_masks.partial, &mask);
+		if (!cpumask_empty(&mask)) {
+			cpu = select_cpu_from_cluster(p, prev_cpu,
+					&mask, &little_distribute_mask_prev);
+			slim_stats_record(SELECT_CPU, 1, 0, 0);
+			if (is_valid_cpu(cpu))
+				return cpu;
+		}
+	}
+
+	if (task_only_blongs_to_cluster(p, BIG)) {
+		cpumask_copy(&mask, iso_masks.big);
+		if (unlikely(b_need_rescue))
+			cpumask_or(&mask, iso_masks.partial, &mask);
+		if (!cpumask_empty(&mask)) {
+			cpu = select_cpu_from_cluster(p, prev_cpu,
+					&mask, &big_distribute_mask_prev);
+			slim_stats_record(SELECT_CPU, 1, 0, 0);
+			if (is_valid_cpu(cpu))
+				return cpu;
+		}
+	}
+
+	cpu = spec_task_before_pick_idle(p, prev_cpu);
+	if (is_valid_cpu(cpu)) {
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		return cpu;
+	}
+
+	/* if the previous CPU is idle, dispatch directly to it */
+	if (test_and_clear_cpu_idle(prev_cpu) || is_valid_cpu(prev_cpu)) {
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		return prev_cpu;
+	}
+
+	cpu = hmbird_pick_idle_cpu(cpu_possible_mask);
+	if (is_valid_cpu(prev_cpu)) {
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		return cpu;
+	}
+
+	if (prev_cpu_misfit(prev_cpu)) {
+		slim_stats_record(SELECT_CPU, 0, 0, 0);
+		cpu = repick_fallback_cpu();
+		if (is_valid_cpu(cpu))
+			return cpu;
+	}
+
+	slim_stats_record(SELECT_CPU, 0, 0, 0);
+	return prev_cpu;
+}
+
+static int select_task_rq_hmbird(struct task_struct *p, int prev_cpu, int wake_flags)
+{
+	return hmbird_select_cpu_dfl(p, prev_cpu, wake_flags);
+}
+
+static void set_cpus_allowed_hmbird(struct task_struct *p, struct affinity_context *ctx)
+{
+	set_cpus_allowed_common(p, ctx);
+}
+
+static void reset_idle_masks(void)
+{
+	cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.little);
+	cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.big);
+	if (is_partial_enabled())
+		cpumask_or(idle_masks.cpu, idle_masks.cpu, iso_masks.partial);
+	hmbird_has_idle_cpus = true;
+}
+
+void __hmbird_update_idle(struct rq *rq, bool idle)
+{
+	int cpu = cpu_of(rq);
+	struct cpumask *sib_mask = topology_sibling_cpumask(cpu);
+
+	if (skip_update_idle())
+		return;
+
+	if (idle) {
+		cpumask_set_cpu(cpu, idle_masks.cpu);
+		if (!hmbird_has_idle_cpus)
+			hmbird_has_idle_cpus = true;
+
+		/*
+		 * idle_masks.smt handling is racy but that's fine as it's only
+		 * for optimization and self-correcting.
+		 */
+		for_each_cpu(cpu, sib_mask) {
+			if (!cpumask_test_cpu(cpu, idle_masks.cpu))
+				return;
+		}
+		cpumask_or(idle_masks.smt, idle_masks.smt, sib_mask);
+	} else {
+		cpumask_clear_cpu(cpu, idle_masks.cpu);
+		if (hmbird_has_idle_cpus && cpumask_empty(idle_masks.cpu))
+			hmbird_has_idle_cpus = false;
+
+		cpumask_andnot(idle_masks.smt, idle_masks.smt, sib_mask);
+	}
+}
+
+#else /* !CONFIG_SMP */
+
+static bool test_and_clear_cpu_idle(int cpu) { return false; }
+static s32 hmbird_pick_idle_cpu(const struct cpumask *cpus_allowed) { return -EBUSY; }
+static void reset_idle_masks(void) {}
+
+#endif /* CONFIG_SMP */
+
+static bool check_rq_for_timeouts(struct rq *rq)
+{
+	struct hmbird_entity *entity;
+	struct task_struct *p;
+	struct rq_flags rf;
+
+	rq_lock_irqsave(rq, &rf);
+	list_for_each_entry(entity, &get_hmbird_rq(rq)->watchdog_list, watchdog_node) {
+		unsigned long last_runnable;
+
+		p = entity->task;
+		last_runnable = get_hmbird_ts(p)->runnable_at;
+
+		if (unlikely(time_after(jiffies,
+				last_runnable + hmbird_watchdog_timeout)) || watchdog_enable) {
+			u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
+
+			rq_unlock_irqrestore(rq, &rf);
+			WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_WDT);
+			HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL,
+					"%-12s[%d] failed to run for %u.%03us, dsq=%llu, mask=%*pb",
+					p->comm, p->pid,
+					dur_ms / 1000, dur_ms % 1000,
+					get_hmbird_ts(p)->dsq ? get_hmbird_ts(p)->dsq->id : 0,
+					cpumask_pr_args(&p->cpus_mask));
+			return 1;
+		}
+	}
+	rq_unlock_irqrestore(rq, &rf);
+	return 0;
+}
+
+static void hmbird_watchdog_workfn(struct work_struct *work)
+{
+	int cpu;
+	bool timeout;
+
+	hmbird_watchdog_timestamp = jiffies;
+
+	for_each_online_cpu(cpu) {
+		timeout = check_rq_for_timeouts(cpu_rq(cpu));
+		if (unlikely(timeout))
+			break;
+
+		cond_resched();
+	}
+	if (!timeout)
+		queue_delayed_work(system_unbound_wq, to_delayed_work(work),
+						hmbird_watchdog_timeout / 2);
+}
+
+static void set_pcp_round(struct rq *rq)
+{
+	int cpu = cpu_of(rq);
+
+	if (atomic64_read(&pcp_dsq_round) != per_cpu(pcp_info, cpu).pcp_seq) {
+		per_cpu(pcp_info, cpu).pcp_seq = atomic64_read(&pcp_dsq_round);
+		per_cpu(pcp_info, cpu).pcp_round = true;
+		hmbird_info_systrace("C|9999|pcp_%d_round|%d\n", cpu, true);
+		per_cpu(pcp_info, cpu).rtime = 0;
+		systrace_output_rtime_state(&per_cpu(pcp_ldsq, cpu),
+						per_cpu(pcp_info, cpu).rtime);
+	}
+}
+
+
+/*
+ * Just for debug: output hmbird on/off state per 10s.
+ */
+#define OUTPUT_INTVAL	(msecs_to_jiffies(10 * 1000))
+static void inform_hmbird_onoff_from_systrace(void)
+{
+	static unsigned long __read_mostly next_print;
+
+	if (time_before(jiffies, READ_ONCE(next_print)))
+		return;
+
+	WRITE_ONCE(next_print, jiffies + OUTPUT_INTVAL);
+	hmbird_output_systrace("C|9999|hmbird_status|%d\n", curr_ss);
+	hmbird_output_systrace("C|9999|parctrl_high_ratio_l|%d\n", parctrl_high_ratio_l);
+	hmbird_output_systrace("C|9999|parctrl_low_ratio_l|%d\n", parctrl_low_ratio_l);
+	hmbird_output_systrace("C|9999|parctrl_high_ratio|%d\n", parctrl_high_ratio);
+	hmbird_output_systrace("C|9999|parctrl_low_ratio|%d\n", parctrl_low_ratio);
+}
+
+void hmbird_notify_sched_tick(void)
+{
+	unsigned long last_check;
+	int cpu = smp_processor_id();
+	struct rq *rq = cpu_rq(cpu);
+
+	hmbird_scheduler_tick();
+
+	if (!hmbird_enabled())
+		return;
+
+	last_check = hmbird_watchdog_timestamp;
+	if (unlikely(time_after(jiffies, last_check + hmbird_watchdog_timeout))) {
+		u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
+
+		HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_STALL,
+				"watchdog failed to check in for %u.%03us",
+				dur_ms / 1000, dur_ms % 1000);
+	}
+	scan_timeout(rq);
+}
+
+static void task_tick_hmbird(struct rq *rq, struct task_struct *curr, int queued)
+{
+	update_curr_hmbird(rq);
+
+	set_pcp_round(rq);
+
+	if (slim_walt_ctrl)
+		hmbird_update_task_ravg_rqclock_wrapper(curr, rq, TASK_UPDATE);
+	/*
+	 * While disabling, always resched and refresh core-sched timestamp as
+	 * we can't trust the slice management or ops.core_sched_before().
+	 */
+	if (hmbird_ops_disabling())
+		get_hmbird_ts(curr)->slice = 0;
+
+	if (!get_hmbird_ts(curr)->slice)
+		resched_curr(rq);
+
+	inform_hmbird_onoff_from_systrace();
+}
+
+static int hmbird_ops_prepare_task(struct task_struct *p, struct task_group *tg)
+{
+	hmbird_cond_deferred_err(TASK_OPS_PREPPED, test_bit(ffs(HMBIRD_TASK_OPS_PREPPED),
+				(unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm);
+
+	get_hmbird_ts(p)->disallow = false;
+
+	hmbird_sched_init_task(p);
+
+	set_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags);
+	set_bit(ffs(HMBIRD_TASK_WATCHDOG_RESET), (unsigned long *)&get_hmbird_ts(p)->flags);
+	return 0;
+}
+
+static void hmbird_ops_enable_task(struct task_struct *p)
+{
+	lockdep_assert_rq_held(task_rq(p));
+	hmbird_cond_deferred_err(TASK_OPS_UNPREPPED, !test_bit(ffs(HMBIRD_TASK_OPS_PREPPED),
+				(unsigned long *)&get_hmbird_ts(p)->flags), "task = %s\n", p->comm);
+
+	clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags);
+	set_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags);
+}
+
+static void hmbird_ops_disable_task(struct task_struct *p)
+{
+	lockdep_assert_rq_held(task_rq(p));
+
+	if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags))
+		clear_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags);
+	else if (test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags))
+		clear_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags);
+}
+
+static void set_task_hmbird_weight(struct task_struct *p)
+{
+	u32 weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
+
+	get_hmbird_ts(p)->weight = hmbird_sched_weight_to_cgroup(weight);
+}
+
+/**
+ * refresh_hmbird_weight - Refresh a task's hmbird weight
+ * @p: task to refresh hmbird weight for
+ *
+ * @get_hmbird_ts(p)->weight carries the task's static priority in cgroup weight scale to
+ * enable easy access from the BPF scheduler. To keep it synchronized with the
+ * current task priority, this function should be called when a new task is
+ * created, priority is changed for a task on hmbird, and a task is switched
+ * to hmbird from other classes.
+ */
+static void refresh_hmbird_weight(struct task_struct *p)
+{
+	lockdep_assert_rq_held(task_rq(p));
+	set_task_hmbird_weight(p);
+}
+
+int hmbird_pre_fork(struct task_struct *p)
+{
+	int ret = 0;
+
+	p->android_oem_data1[HMBIRD_TS_IDX] =
+		(u64)(kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL));
+	if (!get_hmbird_ts(p)) {
+		ret = -1;
+		goto lock;
+	}
+
+	get_hmbird_ts(p)->dsq              = NULL;
+	INIT_LIST_HEAD(&get_hmbird_ts(p)->dsq_node.fifo);
+	RB_CLEAR_NODE(&get_hmbird_ts(p)->dsq_node.priq);
+	INIT_LIST_HEAD(&get_hmbird_ts(p)->watchdog_node);
+	get_hmbird_ts(p)->flags            = 0;
+	get_hmbird_ts(p)->weight           = 0;
+	get_hmbird_ts(p)->sticky_cpu       = -1;
+	get_hmbird_ts(p)->holding_cpu      = -1;
+	get_hmbird_ts(p)->kf_mask          = 0;
+	atomic64_set(&get_hmbird_ts(p)->ops_state, 0);
+	get_hmbird_ts(p)->runnable_at      = INITIAL_JIFFIES;
+	get_hmbird_ts(p)->slice            = HMBIRD_SLICE_DFL;
+	get_hmbird_ts(p)->task             = p;
+	hmbird_set_sched_prop(p, 0);
+
+	get_hmbird_ts(p)->critical_affinity_cpu = -1;
+	get_hmbird_ts(p)->sched_class      = &hmbird_sched_class;
+
+	/*
+	 * BPF scheduler enable/disable paths want to be able to iterate and
+	 * update all tasks which can become complex when racing forks. As
+	 * enable/disable are very cold paths, let's use a percpu_rwsem to
+	 * exclude forks.
+	 */
+lock:
+	percpu_down_read(&hmbird_fork_rwsem);
+
+	return ret;
+}
+
+int hmbird_fork(struct task_struct *p)
+{
+	percpu_rwsem_assert_held(&hmbird_fork_rwsem);
+
+	if (hmbird_enabled())
+		return hmbird_ops_prepare_task(p, task_group(p));
+	else
+		return 0;
+}
+
+void hmbird_post_fork(struct task_struct *p)
+{
+	if (hmbird_enabled()) {
+		struct rq_flags rf;
+		struct rq *rq;
+
+		rq = task_rq_lock(p, &rf);
+		/*
+		 * Set the weight manually before calling ops.enable() so that
+		 * the scheduler doesn't see a stale value if they inspect the
+		 * task struct. We'll invoke ops.set_weight() afterwards, as it
+		 * would be odd to receive a callback on the task before we
+		 * tell the scheduler that it's been fully enabled.
+		 */
+		set_task_hmbird_weight(p);
+		hmbird_ops_enable_task(p);
+		refresh_hmbird_weight(p);
+		task_rq_unlock(rq, p, &rf);
+	}
+
+	spin_lock_irq(&hmbird_tasks_lock);
+	list_add_tail(&get_hmbird_ts(p)->tasks_node, &hmbird_tasks);
+	spin_unlock_irq(&hmbird_tasks_lock);
+
+	percpu_up_read(&hmbird_fork_rwsem);
+}
+
+void hmbird_cancel_fork(struct task_struct *p)
+{
+	struct hmbird_entity *see = get_hmbird_ts(p);
+
+	if (hmbird_enabled())
+		hmbird_ops_disable_task(p);
+
+	kfree(get_hmbird_ts(p));
+	see = NULL;
+
+	percpu_up_read(&hmbird_fork_rwsem);
+}
+
+void hmbird_free(struct task_struct *p)
+{
+	unsigned long flags;
+	struct hmbird_entity *see = get_hmbird_ts(p);
+
+	spin_lock_irqsave(&hmbird_tasks_lock, flags);
+	list_del_init(&get_hmbird_ts(p)->tasks_node);
+	spin_unlock_irqrestore(&hmbird_tasks_lock, flags);
+
+	/*
+	 * @p is off hmbird_tasks and wholly ours. hmbird_ops_enable()'s PREPPED ->
+	 * ENABLED transitions can't race us. Disable ops for @p.
+	 */
+	if (test_bit(ffs(HMBIRD_TASK_OPS_PREPPED), (unsigned long *)&get_hmbird_ts(p)->flags) ||
+		test_bit(ffs(HMBIRD_TASK_OPS_ENABLED), (unsigned long *)&get_hmbird_ts(p)->flags)) {
+		struct rq_flags rf;
+		struct rq *rq;
+
+		rq = task_rq_lock(p, &rf);
+		hmbird_ops_disable_task(p);
+		task_rq_unlock(rq, p, &rf);
+	}
+	kfree(get_hmbird_ts(p));
+	see = NULL;
+}
+
+static void prio_changed_hmbird(struct rq *rq, struct task_struct *p, int oldprio)
+{
+}
+
+static inline bool task_specific_type(uint32_t prop, enum hmbird_task_prop_type type)
+{
+	return (prop >> TOP_TASK_SHIFT) & (1 << type);
+}
+
+static inline enum hmbird_task_prop_type hmbird_get_task_type(struct task_struct *p)
+{
+	uint32_t prop = get_top_task_prop(p);
+
+	if (task_specific_type(prop, HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL))
+		return HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL;
+	if (task_specific_type(prop, HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL))
+		return HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL;
+	if (task_specific_type(prop, HMBIRD_TASK_PROP_PIPELINE))
+		return HMBIRD_TASK_PROP_PIPELINE;
+	if (task_specific_type(prop, HMBIRD_TASK_PROP_COMMON) ||
+			!task_specific_type(prop, HMBIRD_TASK_PROP_DEBUG_OR_LOG)) {
+		return HMBIRD_TASK_PROP_COMMON;
+	}
+	return HMBIRD_TASK_PROP_DEBUG_OR_LOG;
+}
+
+static inline bool hmbird_prio_higher(struct task_struct *a, struct task_struct *b)
+{
+	int type_a = hmbird_get_task_type(a);
+	int type_b = hmbird_get_task_type(b);
+
+	return sched_prop_to_preempt_prio[type_a] > sched_prop_to_preempt_prio[type_b];
+}
+
+static void check_preempt_curr_hmbird(struct rq *rq, struct task_struct *p, int wake_flags)
+{
+	enum cpu_type type;
+	int sp_dl;
+	struct task_struct *curr = NULL;
+
+	switch (hmbird_preempt_policy) {
+	case HMBIRD_PREEMPT_POLICY_PRIO_BASED:
+		curr = rq->curr;
+		if (curr && hmbird_prio_higher(p, curr))
+			goto preempt;
+		break;
+	default:
+		break;
+	}
+	if ((is_pipeline_task(p) && !is_pipeline_task(rq->curr)) ||
+		(is_critical_system_task(p) && !is_critical_system_task(rq->curr)))
+		goto preempt;
+
+	sp_dl = find_idx_from_task(p);
+	if (sp_dl >= SCHED_PROP_DEADLINE_LEVEL1)
+		return;
+
+	type = cpu_cluster(cpu_of(rq));
+	if (type == EXCLUSIVE || ((type == PARTIAL) && !is_partial_enabled()))
+		return;
+
+	if (rq->curr->prio > p->prio)
+		goto preempt;
+
+	return;
+
+preempt:
+	resched_curr(rq);
+}
+
+static void switched_to_hmbird(struct rq *rq, struct task_struct *p) {}
+
+int hmbird_check_setscheduler(struct task_struct *p, int policy)
+{
+	lockdep_assert_rq_held(task_rq(p));
+
+	/* if disallow, reject transitioning into HMBIRD */
+	if (hmbird_enabled() && READ_ONCE(get_hmbird_ts(p)->disallow) &&
+			p->policy != policy && policy == SCHED_HMBIRD)
+		return -EACCES;
+
+	return 0;
+}
+
+#ifdef CONFIG_NO_HZ_FULL
+bool hmbird_can_stop_tick(struct rq *rq)
+{
+	struct task_struct *p = rq->curr;
+
+	if (hmbird_ops_disabling())
+		return false;
+
+	if (p->sched_class != &hmbird_sched_class)
+		return true;
+
+	return get_hmbird_rq(rq)->flags & HMBIRD_RQ_CAN_STOP_TICK;
+}
+#endif
+
+int hmbird_tg_online(struct task_group *tg)
+{
+	struct cgroup *cgrp;
+
+	if (!tg)
+		return 0;
+	cgrp = tg->css.cgroup;
+	if (!cgrp || !(cgrp->kn))
+		return 0;
+	update_cgroup_ids_table(cgrp->kn->id, -1);
+	return 0;
+}
+
+/*
+ * Omitted operations:
+ *
+ * - check_preempt_curr: NOOP as it isn't useful in the wakeup path because the
+ *   task isn't tied to the CPU at that point. Preemption is implemented by
+ *   resetting the victim task's slice to 0 and triggering reschedule on the
+ *   target CPU.
+ *
+ * - migrate_task_rq: Unncessary as task to cpu mapping is transient.
+ *
+ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of
+ *   their current sched_class. Call them directly from sched core instead.
+ *
+ * - task_woken, switched_from: Unnecessary.
+ */
+DEFINE_SCHED_CLASS(hmbird) = {
+	.enqueue_task		= enqueue_task_hmbird,
+	.dequeue_task		= dequeue_task_hmbird,
+	.yield_task		= yield_task_hmbird,
+	.yield_to_task		= yield_to_task_hmbird,
+
+	.check_preempt_curr	= check_preempt_curr_hmbird,
+
+	.pick_next_task		= pick_next_task_hmbird,
+
+	.put_prev_task		= put_prev_task_hmbird,
+	.set_next_task		= set_next_task_hmbird,
+
+#ifdef CONFIG_SMP
+	.balance		= balance_hmbird,
+	.select_task_rq		= select_task_rq_hmbird,
+	.set_cpus_allowed	= set_cpus_allowed_hmbird,
+#endif
+	.task_tick		= task_tick_hmbird,
+
+	.switched_to		= switched_to_hmbird,
+	.prio_changed		= prio_changed_hmbird,
+
+	.update_curr		= update_curr_hmbird,
+
+#ifdef CONFIG_UCLAMP_TASK
+	.uclamp_enabled		= 0,
+#endif
+};
+
+/*
+ * Must with rq lock held.
+ */
+bool task_is_hmbird(struct task_struct *p)
+{
+	return p->sched_class == &hmbird_sched_class;
+}
+
+
+void init_dsq(struct hmbird_dispatch_q *dsq, u64 dsq_id)
+{
+	memset(dsq, 0, sizeof(*dsq));
+
+	raw_spin_lock_init(&dsq->lock);
+	INIT_LIST_HEAD(&dsq->fifo);
+	dsq->id = dsq_id;
+}
+
+static int hmbird_cgroup_init(void)
+{
+	struct cgroup_subsys_state *css;
+
+	css_for_each_descendant_pre(css, &root_task_group.css) {
+		struct task_group *tg = css_tg(css);
+
+		cgrp_dsq_idx_init(css->cgroup, tg);
+	}
+	return 0;
+}
+
+/*
+ * Used by sched_fork() and __setscheduler_prio() to pick the matching
+ * sched_class. dl/rt are already handled.
+ */
+bool task_on_hmbird(struct task_struct *p)
+{
+	return hmbird_enabled();
+}
+
+static void __setscheduler_prio(struct task_struct *p, int prio)
+{
+	bool on_hmbird = task_on_hmbird(p);
+
+	if (p->sched_class == &stop_sched_class) {
+		p->prio = prio;
+		return;
+	} else if (dl_prio(prio))
+		p->sched_class = &dl_sched_class;
+	else if (on_hmbird && rt_prio(prio))
+		p->sched_class = &hmbird_sched_class;
+	else if (rt_prio(prio))
+		p->sched_class = &rt_sched_class;
+	else if (on_hmbird)
+		p->sched_class = &hmbird_sched_class;
+	else
+		p->sched_class = &fair_sched_class;
+
+	p->prio = prio;
+}
+
+/*
+ * Heartbeat, avoid humbird keep running while APP already exit.
+ * Check whether APP send alive-signal periodly.
+ */
+#define HEARTBEAT_TIMEOUT		(msecs_to_jiffies(2500))
+#define HEARTBEAT_CHECK_INTERVAL	(msecs_to_jiffies(1000))
+static struct timer_list hb_timer;
+static unsigned long next_hb;
+
+void hb_timer_handler(struct timer_list *timer)
+{
+	if (!heartbeat_enable)
+		goto refill;
+
+	pr_err("<hmbird_sched>: enter timer.\n");
+	if (heartbeat) {
+		heartbeat = 0;
+		WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT);
+	}
+
+	/* can't detect heartbeat, disable ext. */
+	if (time_after(jiffies, READ_ONCE(next_hb))) {
+		WRITE_ONCE(sw_type, HMBIRD_SWITCH_ERR_HB);
+		HMBIRD_FATAL_INFO_FN(HMBIRD_EXIT_ERROR_HEARTBEAT,
+					"can't detect heartbeat, disable ext\n");
+	}
+refill:
+	mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL);
+}
+
+static void hb_timer_start(void)
+{
+	mod_timer(&hb_timer, jiffies + HEARTBEAT_CHECK_INTERVAL);
+}
+
+static void hb_timer_init(void)
+{
+	timer_setup(&hb_timer, hb_timer_handler, 0);
+}
+
+static void hb_timer_exit(void)
+{
+	del_timer(&hb_timer);
+}
+
+static bool check_and_disable_cpuhp(void)
+{
+	struct rq *rq;
+	struct rq_flags rf;
+	int cpu;
+
+	cpu_hotplug_disable();
+	cpus_read_lock();
+	for_each_possible_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		rq_lock_irqsave(rq, &rf);
+		if (!rq->online) {
+			rq_unlock_irqrestore(rq, &rf);
+			goto err_offline;
+		}
+		rq_unlock_irqrestore(rq, &rf);
+	}
+	cpus_read_unlock();
+	return true;
+
+err_offline:
+	cpus_read_unlock();
+	cpu_hotplug_enable();
+	return false;
+}
+
+static void reenable_cpuhp(void)
+{
+	cpu_hotplug_enable();
+}
+
+static void scheduler_switch_done(bool final_state)
+{
+	/* set scx_enable again, switch may fail. */
+	scx_enable = final_state;
+	/* reset heartbeat*/
+	WRITE_ONCE(next_hb, jiffies + HEARTBEAT_TIMEOUT);
+
+	if (final_state)
+		hb_timer_start();
+	else
+		hb_timer_exit();
+}
+
+/**
+ * ss : curr switch state
+ * finish : success or fail
+ * enable : curr operation is diabling or enabling?
+ * fail_reason : string output when failed, empty string when success.
+ */
+static void hmbird_switch_log(enum switch_stat ss, bool finish, bool enable, char *fail_reason)
+{
+	char *s1 = finish ? "finished" : "failed";
+	char *s2 = enable ? "enabled" : "disabled";
+
+	hmbird_internal_systrace("C|9999|hmbird_status|%d\n", ss);
+	if (ss == HMBIRD_DISABLED || ss == HMBIRD_ENABLED) {
+		sw_update(md_info, jiffies, finish, ss, READ_ONCE(sw_type));
+		hmbird_debug("hmbird %s %s at jiffies = %lu, clock = %lu, reason = %s\n",
+				s2, s1, jiffies, (unsigned long)sched_clock(), fail_reason);
+	}
+	curr_ss = ss;
+}
+
+bool get_hmbird_ops_enabled(void)
+{
+	return atomic_read(&__hmbird_ops_enabled);
+}
+
+bool get_non_hmbird_task(void)
+{
+	return atomic_read(&non_hmbird_task);
+}
+
+static void hmbird_ops_disable_workfn(struct kthread_work *work)
+{
+	struct hmbird_task_iter sti;
+	struct task_struct *p;
+	int cpu;
+
+	hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 0, "");
+	cancel_delayed_work_sync(&hmbird_watchdog_work);
+
+	mutex_lock(&hmbird_ops_enable_mutex);
+	switch (hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLING)) {
+	case HMBIRD_OPS_DISABLED:
+		WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) !=
+					HMBIRD_OPS_DISABLING);
+		hmbird_switch_log(HMBIRD_DISABLED, 0, 0, "already disabled");
+		scheduler_switch_done(false);
+		mutex_unlock(&hmbird_ops_enable_mutex);
+		return;
+	case HMBIRD_OPS_PREPPING:
+		fallthrough;
+	case HMBIRD_OPS_DISABLING:
+		/* shouldn't happen but handle it like ENABLING if it does */
+		WARN_ONCE(true, "hmbird: duplicate disabling instance?");
+		fallthrough;
+	case HMBIRD_OPS_ENABLING:
+	case HMBIRD_OPS_ENABLED:
+		break;
+	}
+
+	/* kick all CPUs to restore ticks */
+	for_each_possible_cpu(cpu)
+		resched_cpu(cpu);
+
+	/* avoid racing against fork and cgroup changes */
+	cpus_read_lock();
+	percpu_down_write(&hmbird_fork_rwsem);
+
+	hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 0, "");
+	spin_lock_irq(&hmbird_tasks_lock);
+	atomic_set(&__hmbird_ops_enabled, false);
+	hmbird_task_iter_init(&sti);
+	while ((p = hmbird_task_iter_next_filtered_locked(&sti))) {
+		const struct sched_class *old_class = p->sched_class;
+		struct rq *rq = task_rq(p);
+		bool alive = READ_ONCE(p->__state) != TASK_DEAD;
+
+		update_rq_clock(rq);
+
+		SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE |
+					DEQUEUE_NOCLOCK) {
+			get_hmbird_ts(p)->slice = min_t(u64,
+					get_hmbird_ts(p)->slice, HMBIRD_SLICE_DFL);
+
+			__setscheduler_prio(p, p->prio);
+		}
+
+		if (alive)
+			check_class_changed(task_rq(p), p, old_class, p->prio);
+
+		hmbird_ops_disable_task(p);
+	}
+	hmbird_task_iter_exit(&sti);
+	spin_unlock_irq(&hmbird_tasks_lock);
+	hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 0, "");
+
+	atomic_set(&non_hmbird_task, true);
+	/* no task is on hmbird, turn off all the switches and flush in-progress calls */
+	static_branch_disable_cpuslocked(&hmbird_ops_cpu_preempt);
+	synchronize_rcu();
+
+	percpu_up_write(&hmbird_fork_rwsem);
+	cpus_read_unlock();
+
+	if (slim_walt_ctrl)
+		slim_walt_enable(false);
+
+	mutex_unlock(&hmbird_ops_enable_mutex);
+
+	WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_DISABLED) !=
+			HMBIRD_OPS_DISABLING);
+	hmbird_switch_log(HMBIRD_DISABLED, 1, 0, "");
+	scheduler_switch_done(false);
+	reenable_cpuhp();
+}
+
+static DEFINE_KTHREAD_WORK(hmbird_ops_disable_work, hmbird_ops_disable_workfn);
+
+static void schedule_hmbird_ops_disable_work(void)
+{
+	struct kthread_worker *helper = READ_ONCE(hmbird_ops_helper);
+
+	/*
+	 * We may be called spuriously before the first bpf_hmbird_reg(). If
+	 * hmbird_ops_helper isn't set up yet, there's nothing to do.
+	 */
+	if (helper)
+		kthread_queue_work(helper, &hmbird_ops_disable_work);
+}
+
+static void hmbird_ops_disable(void)
+{
+	schedule_hmbird_ops_disable_work();
+}
+
+static void hmbird_err_exit_workfn(struct work_struct *work)
+{
+	int cpu;
+	struct cpufreq_policy *policy;
+
+	hmbird_ctrl(false);
+
+	for_each_present_cpu(cpu) {
+		policy = cpufreq_cpu_get(cpu);
+		if (!policy)
+			continue;
+		if (cpu != policy->cpu)
+			goto put;
+		down_write(&policy->rwsem);
+		WARN_ON(store_scaling_governor(policy,
+				saved_gov[cpu], strlen(saved_gov[cpu])) <= 0);
+		up_write(&policy->rwsem);
+		hmbird_info_trace("<heartbeat>:restore origin gov : %s\n", saved_gov[cpu]);
+put:
+		cpufreq_cpu_put(policy);
+	}
+	memset((char *)saved_gov, 0, sizeof(saved_gov));
+}
+
+void hmbird_ops_exit(void)
+{
+	queue_work(system_unbound_wq, &hmbird_err_exit_work);
+}
+
+static struct kthread_worker *hmbird_create_rt_helper(const char *name)
+{
+	struct kthread_worker *helper;
+
+	helper = kthread_create_worker(0, name);
+	if (helper)
+		sched_set_fifo(helper->task);
+	return helper;
+}
+
+static inline void set_audio_thread_sched_prop(struct task_struct *p)
+{
+	struct cgroup_subsys_state *css;
+
+	if (likely(p->prio >= MAX_RT_PRIO))
+		return;
+
+	rcu_read_lock();
+	css = task_css(p, cpuset_cgrp_id);
+	if (!css) {
+		rcu_read_unlock();
+		return;
+	}
+	rcu_read_unlock();
+
+	if (!strcmp(css->cgroup->kn->name, "audio-app"))
+		hmbird_set_sched_prop(p, SCHED_PROP_DEADLINE_LEVEL1);
+}
+
+static int hmbird_ops_enable(void *unused)
+{
+	struct hmbird_task_iter sti;
+	struct task_struct *p;
+	int ret;
+	int tcnt = 0;
+	unsigned long long start = 0;
+
+	if (!check_and_disable_cpuhp()) {
+		hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "cpu offline");
+		return -EBUSY;
+	}
+	hmbird_switch_log(HMBIRD_SWITCH_PREP, 0, 1, "");
+	mutex_lock(&hmbird_ops_enable_mutex);
+
+	if (!hmbird_ops_helper) {
+		WRITE_ONCE(hmbird_ops_helper,
+				hmbird_create_rt_helper("hmbird_ops_helper"));
+		if (!hmbird_ops_helper) {
+			ret = -ENOMEM;
+			goto err_unlock;
+		}
+	}
+
+	if (hmbird_ops_enable_state() != HMBIRD_OPS_DISABLED) {
+		ret = -EBUSY;
+		goto err_unlock;
+	}
+
+	WARN_ON_ONCE(hmbird_ops_set_enable_state(HMBIRD_OPS_PREPPING) !=
+				HMBIRD_OPS_DISABLED);
+
+	hmbird_warned_zero_slice = false;
+
+	atomic64_set(&hmbird_nr_rejected, 0);
+
+	/*
+	 * Keep CPUs stable during enable so that the BPF scheduler can track
+	 * online CPUs by watching ->on/offline_cpu() after ->init().
+	 */
+	cpus_read_lock();
+
+	hmbird_watchdog_timeout = HMBIRD_WATCHDOG_MAX_TIMEOUT;
+
+	hmbird_watchdog_timestamp = jiffies;
+	queue_delayed_work(system_unbound_wq, &hmbird_watchdog_work,
+				hmbird_watchdog_timeout / 2);
+
+	/*
+	 * Lock out forks, cgroup on/offlining and moves before opening the
+	 * floodgate so that they don't wander into the operations prematurely.
+	 */
+	percpu_down_write(&hmbird_fork_rwsem);
+
+	reset_idle_masks();
+
+	/*
+	 * All cgroups should be initialized before letting in tasks. cgroup
+	 * on/offlining and task migrations are already locked out.
+	 */
+	ret = hmbird_cgroup_init();
+	if (ret)
+		goto err_disable_unlock;
+
+	/*
+	 * Enable ops for every task. Fork is excluded by hmbird_fork_rwsem
+	 * preventing new tasks from being added. No need to exclude tasks
+	 * leaving as hmbird_free() can handle both prepped and enabled
+	 * tasks. Prep all tasks first and then enable them with preemption
+	 * disabled.
+	 */
+	spin_lock_irq(&hmbird_tasks_lock);
+
+	atomic_set(&non_hmbird_task, false);
+	atomic_set(&__hmbird_ops_enabled, true);
+
+	hmbird_task_iter_init(&sti);
+	while ((p = hmbird_task_iter_next_filtered(&sti)))
+		hmbird_ops_prepare_task(p, task_group(p));
+
+	hmbird_task_iter_exit(&sti);
+
+	/*
+	 * All tasks are prepped but are still ops-disabled. Ensure that
+	 * %current can't be scheduled out and switch everyone.
+	 * preempt_disable() is necessary because we can't guarantee that
+	 * %current won't be starved if scheduled out while switching.
+	 */
+	preempt_disable();
+
+	/*
+	 * From here on, the disable path must assume that tasks have ops
+	 * enabled and need to be recovered.
+	 */
+	if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLING, HMBIRD_OPS_PREPPING)) {
+		atomic_set(&non_hmbird_task, true);
+		atomic_set(&__hmbird_ops_enabled, false);
+		preempt_enable();
+		spin_unlock_irq(&hmbird_tasks_lock);
+		ret = -EBUSY;
+		goto err_disable_unlock;
+	}
+
+	/*
+	 * We're fully committed and can't fail. The PREPPED -> ENABLED
+	 * transitions here are synchronized against hmbird_free() through
+	 * hmbird_tasks_lock.
+	 */
+	start = sched_clock();
+	hmbird_switch_log(HMBIRD_RQ_SWITCH_BEGIN, 0, 1, "");
+	hmbird_task_iter_init(&sti);
+	while ((p = hmbird_task_iter_next_filtered_locked(&sti))) {
+		tcnt++;
+		if (READ_ONCE(p->__state) != TASK_DEAD) {
+			const struct sched_class *old_class = p->sched_class;
+			struct rq *rq = task_rq(p);
+
+			set_audio_thread_sched_prop(p);
+			update_rq_clock(rq);
+
+			SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_MOVE |
+						DEQUEUE_NOCLOCK) {
+				hmbird_ops_enable_task(p);
+				__setscheduler_prio(p, p->prio);
+			}
+
+			check_class_changed(task_rq(p), p, old_class, p->prio);
+		} else {
+			hmbird_ops_disable_task(p);
+		}
+	}
+	hmbird_task_iter_exit(&sti);
+
+	spin_unlock_irq(&hmbird_tasks_lock);
+	hmbird_switch_log(HMBIRD_RQ_SWITCH_DONE, 0, 1, "");
+	preempt_enable();
+	percpu_up_write(&hmbird_fork_rwsem);
+
+	if (!hmbird_ops_tryset_enable_state(HMBIRD_OPS_ENABLED, HMBIRD_OPS_ENABLING)) {
+		ret = -EBUSY;
+		goto err_disable;
+	}
+
+	cpus_read_unlock();
+	mutex_unlock(&hmbird_ops_enable_mutex);
+	hmbird_switch_log(HMBIRD_ENABLED, 1, 1, "");
+	scheduler_switch_done(true);
+
+	return 0;
+
+err_unlock:
+	mutex_unlock(&hmbird_ops_enable_mutex);
+	hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_unlock");
+	scheduler_switch_done(false);
+	return ret;
+
+err_disable_unlock:
+	percpu_up_write(&hmbird_fork_rwsem);
+err_disable:
+	cpus_read_unlock();
+	mutex_unlock(&hmbird_ops_enable_mutex);
+	/* must be fully disabled before returning */
+	hmbird_ops_disable();
+	kthread_flush_work(&hmbird_ops_disable_work);
+	hmbird_switch_log(HMBIRD_DISABLED, 0, 1, "err_disable");
+	scheduler_switch_done(false);
+	return ret;
+}
+
+#ifdef CONFIG_SCHED_DEBUG
+static const char *hmbird_ops_enable_state_str[] = {
+	[HMBIRD_OPS_PREPPING]	= "prepping",
+	[HMBIRD_OPS_ENABLING]	= "enabling",
+	[HMBIRD_OPS_ENABLED]	= "enabled",
+	[HMBIRD_OPS_DISABLING]	= "disabling",
+	[HMBIRD_OPS_DISABLED]	= "disabled",
+};
+
+static int hmbird_debug_show(struct seq_file *m, void *v)
+{
+	mutex_lock(&hmbird_ops_enable_mutex);
+	seq_printf(m, "%-30s: %d\n", "enabled", hmbird_enabled());
+	seq_printf(m, "%-30s: %s\n", "enable_state",
+			hmbird_ops_enable_state_str[hmbird_ops_enable_state()]);
+	seq_printf(m, "%-30s: %llu\n", "nr_rejected",
+			atomic64_read(&hmbird_nr_rejected));
+	mutex_unlock(&hmbird_ops_enable_mutex);
+	return 0;
+}
+
+static int hmbird_debug_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, hmbird_debug_show, NULL);
+}
+
+const struct file_operations sched_hmbird_fops = {
+	.open		= hmbird_debug_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif
+
+
+static int bpf_hmbird_reg(void *kdata)
+{
+	return hmbird_ops_enable(kdata);
+}
+
+static int bpf_hmbird_unreg(void *kdata)
+{
+	hmbird_ops_disable();
+	kthread_flush_work(&hmbird_ops_disable_work);
+	return 0;
+}
+
+void set_hmbird_module_loaded(int is_loaded)
+{
+	atomic_set(&hmbird_module_loaded, is_loaded);
+}
+
+/*
+ * MUST load hmbird module before enable hmbird scheduler
+ * load track & hmbird gover implemented in hmbird module
+ */
+int hmbird_ctrl(bool enable)
+{
+	if (!atomic_read(&hmbird_module_loaded)) {
+		hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED,
+					0, enable, "ext module unloaded\n");
+		return -EINVAL;
+	}
+
+	if (enable && (hmbird_ops_enable_state() == HMBIRD_OPS_ENABLED
+			|| hmbird_ops_enable_state() == HMBIRD_OPS_ENABLING
+			|| hmbird_ops_enable_state() == HMBIRD_OPS_PREPPING)) {
+		/* Executing or completed, no need to repeat. */
+		hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED,
+					0, enable, "already enabled(ing)\n");
+		hmbird_err(ALREADY_ENABLED, "ext already in enable state, exit!\n");
+		return -EBUSY;
+	}
+	if (!enable && (hmbird_ops_enable_state() == HMBIRD_OPS_DISABLING ||
+			hmbird_ops_enable_state() == HMBIRD_OPS_DISABLED)) {
+		/* Executing or completed, no need to repeat. */
+		hmbird_switch_log(hmbird_enabled() ? HMBIRD_ENABLED : HMBIRD_DISABLED,
+					0, enable, "already disabled(ing)\n");
+		hmbird_err(ALREADY_DISABLED, "ext already in disable state, exit!\n");
+		return -EBUSY;
+	}
+	if (enable)
+		return bpf_hmbird_reg(NULL);
+	else
+		return bpf_hmbird_unreg(NULL);
+}
+
+void set_cpu_isomask(int cpu, cpumask_var_t *mask)
+{
+	cpumask_clear_cpu(cpu, iso_masks.ex_free);
+	cpumask_clear_cpu(cpu, iso_masks.exclusive);
+	cpumask_clear_cpu(cpu, iso_masks.partial);
+	cpumask_clear_cpu(cpu, iso_masks.big);
+	cpumask_clear_cpu(cpu, iso_masks.little);
+	cpumask_set_cpu(cpu, *mask);
+}
+
+void set_cpu_cluster(u64 cpu_cluster)
+{
+	int cpu;
+
+	for_each_present_cpu(cpu) {
+		u64 pos = 1 << cpu;
+
+		if ((pos & cpu_cluster) != 0) {
+			set_cpu_isomask(cpu, &(iso_masks.ex_free));
+			continue;
+		} else
+			pos = pos << 8;
+		if ((pos & cpu_cluster) != 0) {
+			set_cpu_isomask(cpu, &(iso_masks.exclusive));
+			continue;
+		} else
+			pos = pos << 8;
+		if ((pos & cpu_cluster) != 0) {
+			set_cpu_isomask(cpu, &(iso_masks.partial));
+			continue;
+		} else
+			pos = pos << 8;
+		if ((pos & cpu_cluster) != 0) {
+			set_cpu_isomask(cpu, &(iso_masks.big));
+			continue;
+		} else
+			pos = pos << 8;
+		if ((pos & cpu_cluster) != 0)
+			set_cpu_isomask(cpu, &(iso_masks.little));
+	}
+}
+
+static void get_hmbird_snapshot(struct panic_snapshot_t *p)
+{
+	struct hmbird_dispatch_q *dsq;
+	struct rq *rq;
+	int cpu, i;
+
+	for_each_possible_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		p->rq_nr[cpu] = rq->nr_running;
+		p->scxrq_nr[cpu] = get_hmbird_rq(rq)->nr_running;
+	}
+
+	for (i = 0; i < MAX_GLOBAL_DSQS; i++) {
+		struct hmbird_entity *entity;
+
+		dsq = &gdsqs[i];
+		raw_spin_lock(&dsq->lock);
+		if (list_empty(&dsq->fifo)) {
+			raw_spin_unlock(&dsq->lock);
+			continue;
+		}
+
+		entity = list_first_entry(&dsq->fifo, struct hmbird_entity, dsq_node.fifo);
+		if (!entity) {
+			raw_spin_unlock(&dsq->lock);
+			continue;
+		}
+		p->runnable_at[i] = entity->runnable_at;
+		raw_spin_unlock(&dsq->lock);
+
+	}
+
+	p->snap_misc.hmbird_enabled = hmbird_enabled();
+	p->snap_misc.curr_ss = curr_ss;
+	p->snap_misc.hmbird_ops_enable_state_var = (u64)hmbird_ops_enable_state();
+	p->snap_misc.parctrl_high_ratio = parctrl_high_ratio;
+	p->snap_misc.parctrl_low_ratio = parctrl_low_ratio;
+	p->snap_misc.parctrl_high_ratio_l = parctrl_high_ratio_l;
+	p->snap_misc.parctrl_low_ratio_l = parctrl_low_ratio_l;
+	p->snap_misc.isoctrl_high_ratio = isoctrl_high_ratio;
+	p->snap_misc.isoctrl_low_ratio = isoctrl_low_ratio;
+	p->snap_misc.misfit_ds = misfit_ds;
+	p->snap_misc.partial_enable = partial_enable;
+	p->snap_misc.iso_free_rescue = iso_free_rescue;
+	p->snap_misc.isolate_ctrl = isolate_ctrl;
+	p->snap_misc.snap_jiffies = jiffies;
+	p->snap_misc.snap_time = local_clock();
+}
+
+// MTK minidump begin
+static void init_desc_meta(struct meta_desc_t *m, char *str, u64 d1, u64 d2, u64 d3)
+{
+	strscpy(m->desc_str, str, DESC_STR_LEN);
+	m->len = d1 * d2 * d3;
+	m->parse[0] = d1;
+	m->parse[1] = d2;
+	m->parse[2] = d3;
+}
+
+static void init_desc_metas(struct md_info_t *m)
+{
+	init_desc_meta(&m->kern_dump.sw_rec_meta, "switch record :",
+			1, MAX_SWITCHS, SWITCH_ITEMS);
+
+	init_desc_meta(&m->kern_dump.sw_idx_meta, "switch idx :", 1, 1, 1);
+
+	init_desc_meta(&m->kern_dump.excep_rec_meta,
+			"excep record :", 1, MAX_EXCEP_ID, MAX_EXCEPS);
+
+	init_desc_meta(&m->kern_dump.excep_idx_meta,
+			"excep idx :", 1, 1, MAX_EXCEP_ID);
+
+	init_desc_meta(&m->kern_dump.snap.runnable_at_meta,
+			"each dsq runnable at :", 1, 1, MAX_GLOBAL_DSQS);
+
+	init_desc_meta(&m->kern_dump.snap.rq_nr_meta,
+			"rq runnable task nr:", 1, 1, num_possible_cpus());
+
+	init_desc_meta(&m->kern_dump.snap.scxrq_nr_meta,
+			"scxrq runnable task nr :", 1, 1, num_possible_cpus());
+
+	init_desc_meta(&m->kern_dump.snap.snap_misc_meta,
+			"misc snap params :", 1, 1, SNAP_ITEMS);
+}
+
+static void init_md_meta(struct md_info_t *m)
+{
+	m->meta.desc_meta_len = sizeof(struct meta_desc_t) / sizeof(u64);
+	m->meta.desc_str_len = DESC_STR_LEN / sizeof(u64);
+	m->meta.unit_size = sizeof(u64);
+	m->meta.switches = MAX_SWITCHS;
+	m->meta.exceps = MAX_EXCEPS;
+	m->meta.global_dsqs = MAX_GLOBAL_DSQS;
+	m->meta.parse_dimens = PARSE_DIMENS;
+	m->meta.nr_cpus = num_possible_cpus();
+	m->meta.real_cpus = nr_cpu_ids;
+	m->meta.self_len = sizeof(struct md_meta_t) / sizeof(u64);
+	m->meta.nr_meta_desc = 8;
+	m->meta.dump_real_size = sizeof(struct md_info_t) / sizeof(u64);
+
+	init_desc_metas(m);
+}
+
+#define MINIDUMP_DFL_SIZE	(4 * 1024)
+
+struct notifier_block hmbird_panic_blk;
+static int hmbird_panic_handler(struct notifier_block *this,
+					unsigned long event, void *ptr)
+{
+	if (!md_info)
+		return NOTIFY_DONE;
+
+	get_hmbird_snapshot(&md_info->kern_dump.snap);
+
+	return NOTIFY_DONE;
+}
+
+static void panic_blk_init(void)
+{
+	int dump_size = max_t(u32, sizeof(struct md_info_t), MINIDUMP_DFL_SIZE);
+
+	md_info = kzalloc(dump_size, GFP_KERNEL);
+	if (!md_info)
+		return;
+	init_md_meta(md_info);
+
+	hmbird_panic_blk.notifier_call = hmbird_panic_handler;
+	/* make sure to execute before minidump. */
+	hmbird_panic_blk.priority = INT_MAX;
+	atomic_notifier_chain_register(&panic_notifier_list, &hmbird_panic_blk);
+
+	hmbird_debug("register minidump.\n");
+}
+
+void hmbird_get_md_info(unsigned long *vaddr, unsigned long *size)
+{
+	*vaddr = (unsigned long)md_info;
+	*size = sizeof(struct md_info_t);
+}
+// MTK minidump end
+
+static inline void init_sched_prop_to_preempt_prio(void)
+{
+	for (int i = 0; i < HMBIRD_TASK_PROP_MAX; i++) {
+		switch (i) {
+		case HMBIRD_TASK_PROP_TRANSIENT_AND_CRITICAL:
+			sched_prop_to_preempt_prio[i] = 5;
+			break;
+
+		case HMBIRD_TASK_PROP_PERIODIC_AND_CRITICAL:
+			sched_prop_to_preempt_prio[i] = 4;
+			break;
+
+		case HMBIRD_TASK_PROP_PIPELINE:
+		case HMBIRD_TASK_PROP_ISOLATE:
+			sched_prop_to_preempt_prio[i] = 3;
+			break;
+
+		case HMBIRD_TASK_PROP_COMMON:
+			sched_prop_to_preempt_prio[i] = 1;
+			break;
+
+		case HMBIRD_TASK_PROP_DEBUG_OR_LOG:
+			sched_prop_to_preempt_prio[i] = 0;
+			break;
+
+		default:
+			sched_prop_to_preempt_prio[i] = 2;
+			break;
+		}
+	}
+}
+
+void __init init_sched_hmbird_class(void)
+{
+	int cpu;
+	u32 v;
+	struct hmbird_entity *init_hmbird;
+
+	/*
+	 * The following is to prevent the compiler from optimizing out the enum
+	 * definitions so that BPF scheduler implementations can use them
+	 * through the generated vmlinux.h.
+	 */
+	WRITE_ONCE(v, HMBIRD_WAKE_EXEC | HMBIRD_ENQ_WAKEUP | HMBIRD_DEQ_SLEEP |
+			HMBIRD_KICK_PREEMPT);
+
+	init_dsq(&hmbird_dsq_global, HMBIRD_DSQ_GLOBAL);
+	init_dsq_at_boot();
+	init_isolate_cpus();
+	hb_timer_init();
+	init_sched_prop_to_preempt_prio();
+#ifdef CONFIG_SMP
+	WARN_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL));
+	WARN_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL));
+#endif
+
+	/*
+	 * we can't static init init_task's hmbird struct, init here.
+	 * init_task->hmbird would not use during boot.
+	 */
+	init_hmbird = kmalloc(sizeof(struct hmbird_entity), GFP_KERNEL);
+	init_task.android_oem_data1[HMBIRD_TS_IDX] = (u64)init_hmbird;
+	if (init_hmbird) {
+		INIT_LIST_HEAD(&init_hmbird->dsq_node.fifo);
+		INIT_LIST_HEAD(&init_hmbird->watchdog_node);
+		init_hmbird->sticky_cpu = -1;
+		init_hmbird->holding_cpu = -1;
+		atomic64_set(&init_hmbird->ops_state, 0);
+		init_hmbird->runnable_at = jiffies;
+		init_hmbird->slice = HMBIRD_SLICE_DFL;
+		init_hmbird->task = &init_task;
+		hmbird_set_sched_prop(&init_task, 0);
+	} else {
+		hmbird_err(INIT_TASK_FAIL, "<fatal>:alloc init_task.scx failed!!!\n");
+	}
+
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		/*
+		 * exec during boot phase, no need to care about alloc failed.
+		 * lifecycle same to rq, no need to free.
+		 */
+		rq->android_oem_data1[HMBIRD_RQ_IDX] =
+			(u64)kmalloc(sizeof(struct hmbird_rq), GFP_KERNEL);
+		if (get_hmbird_rq(rq)) {
+			get_hmbird_rq(rq)->rq = rq;
+			get_hmbird_rq(rq)->srq = &per_cpu(hmbird_sched_rq_stats, cpu);
+			get_hmbird_rq(rq)->srq->sched_ravg_window_ptr = &hmbird_sched_ravg_window;
+		} else {
+			hmbird_err(ALLOC_RQSCX_FAIL, "<fatal>:alloc rq->scx failed!!!\n");
+		}
+
+		rq->android_oem_data1[HMBIRD_OPS_IDX] =
+				(u64)kzalloc(sizeof(struct hmbird_ops), GFP_KERNEL);
+		if (!get_hmbird_ops(rq))
+			pr_err("fatal error : alloc get_hmbird_ops(rq) failed!!!\n");
+		init_dsq(&get_hmbird_rq(rq)->local_dsq, HMBIRD_DSQ_LOCAL);
+		INIT_LIST_HEAD(&get_hmbird_rq(rq)->watchdog_list);
+
+		WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_kick, GFP_KERNEL));
+		WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_preempt, GFP_KERNEL));
+		WARN_ON(!zalloc_cpumask_var(&get_hmbird_rq(rq)->cpus_to_wait, GFP_KERNEL));
+
+		hmbird_ops_init(get_hmbird_ops(rq));
+	}
+
+	INIT_DELAYED_WORK(&hmbird_watchdog_work, hmbird_watchdog_workfn);
+	INIT_WORK(&hmbird_err_exit_work, hmbird_err_exit_workfn);
+	hmbird_misc_init();
+
+	panic_blk_init();
+}
+
diff --git a/kernel/sched/hmbird/hmbird_misc.c b/kernel/sched/hmbird/hmbird_misc.c
new file mode 100755
index 000000000000..52582c22052c
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_misc.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/*
+ * @Description:
+ * @Version:
+ * @Author: wangrui8
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+
+#include <linux/kprobes.h>
+#include "hmbird_sched.h"
+#include <linux/sched/hmbird.h>
+#include <linux/kernel.h>
+#include "slim.h"
+
+noinline int tracing_mark_write(const char *buf)
+{
+	trace_printk(buf);
+	return 0;
+}
+
+struct yield_opt_params yield_opt_params = {
+	.enable = 0,
+	.frame_per_sec = 120,
+	.frame_time_ns = NSEC_PER_SEC / 120,
+	.yield_headroom = 10,
+};
+
+DEFINE_PER_CPU(struct sched_yield_state, ystate);
+
+static inline void hmbird_yield_state_update(struct sched_yield_state *ys)
+{
+	if (!raw_spin_is_locked(&ys->lock))
+		return;
+	int yield_headroom = yield_opt_params.yield_headroom;
+
+	if (ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH || ys->sleep_times > 1
+						|| ys->yield_cnt_after_sleep > yield_headroom) {
+		ys->sleep = min(ys->sleep + yield_headroom * YIELD_DURATION, MAX_YIELD_SLEEP);
+	} else if (!ys->yield_cnt && (ys->sleep_times == 1) && !ys->yield_cnt_after_sleep) {
+		ys->sleep = max(ys->sleep - yield_headroom * YIELD_DURATION, MIN_YIELD_SLEEP);
+	}
+	ys->yield_cnt = 0;
+	ys->sleep_times = 0;
+	ys->yield_cnt_after_sleep = 0;
+}
+
+void hmbird_skip_yield(long *skip)
+{
+	if (!get_hmbird_ops_enabled() || !yield_opt_params.enable)
+		return;
+	unsigned long flags, sleep_now = 0;
+	struct sched_yield_state *ys;
+	int cpu = raw_smp_processor_id(), cont_yield, new_frame;
+	int frame_time_ns = yield_opt_params.frame_time_ns;
+	int yield_headroom = yield_opt_params.yield_headroom;
+	u64 wc;
+
+	if (!(*skip)) {
+		wc = sched_clock();
+		ys = &per_cpu(ystate, cpu);
+		raw_spin_lock_irqsave(&ys->lock, flags);
+
+		cont_yield = (wc - ys->last_yield_time) < MIN_YIELD_SLEEP;
+		new_frame = (wc - ys->last_update_time) > (frame_time_ns >> 1);
+
+		if (!cont_yield && new_frame) {
+			hmbird_yield_state_update(ys);
+			ys->last_update_time = wc;
+			ys->sleep_end = ys->last_yield_time + frame_time_ns
+						- yield_headroom * YIELD_DURATION;
+		}
+
+		if (ys->sleep > MIN_YIELD_SLEEP || ys->yield_cnt >= DEFAULT_YIELD_SLEEP_TH) {
+			*skip = true;
+
+			sleep_now = ys->sleep_times ?
+				max(ys->sleep >> ys->sleep_times, MIN_YIELD_SLEEP):ys->sleep;
+			if (wc + sleep_now > ys->sleep_end) {
+				u64 delta = ys->sleep_end - wc;
+
+				if (ys->sleep_end > wc && delta > 3 * YIELD_DURATION)
+					sleep_now = delta;
+				else
+					sleep_now = 0;
+			}
+			raw_spin_unlock_irqrestore(&ys->lock, flags);
+			if (sleep_now) {
+				sleep_now = div64_u64(sleep_now, 1000);
+				usleep_range_state(sleep_now, sleep_now, TASK_IDLE);
+			}
+			ys->sleep_times++;
+			ys->last_yield_time = sched_clock();
+			return;
+		}
+		if (ys->sleep_times)
+			ys->yield_cnt_after_sleep++;
+		else
+			(ys->yield_cnt)++;
+		ys->last_yield_time = wc;
+		raw_spin_unlock_irqrestore(&ys->lock, flags);
+	}
+}
+
+void hmbird_ops_init(struct hmbird_ops *hmbird_ops)
+{
+	hmbird_ops->scx_enable = get_hmbird_ops_enabled;
+	hmbird_ops->check_non_task = get_non_hmbird_task;
+	hmbird_ops->hmbird_get_md_info = hmbird_get_md_info;
+}
+
+void hmbird_misc_init(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct sched_yield_state *ys = &per_cpu(ystate, cpu);
+
+		raw_spin_lock_init(&ys->lock);
+	}
+
+	set_hmbird_module_loaded(1);
+}
+
diff --git a/kernel/sched/hmbird/hmbird_sched.h b/kernel/sched/hmbird/hmbird_sched.h
new file mode 100755
index 000000000000..6b5ef43cb827
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_sched.h
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+#ifndef __HMBIRD_SCHED__
+#define __HMBIRD_SCHED__
+
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/smp.h>
+#include <linux/tick.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#include <linux/irq_work.h>
+#include <linux/cpufreq.h>
+#include <linux/kthread.h>
+#include <linux/sched/clock.h>
+#include <uapi/linux/sched/types.h>
+#include "../../time/tick-sched.h"
+#include "../../sched/sched.h"
+#include <trace/hooks/sched.h>
+#include <linux/delay.h>
+#include <linux/sched/hmbird.h>
+
+#define REGISTER_TRACE_VH(vender_hook, handler) \
+{ \
+	ret = register_trace_##vender_hook(handler, NULL); \
+	if (ret) { \
+		pr_err("failed to register_trace_"#vender_hook", ret=%d\n", ret); \
+	} \
+}
+#define REGISTER_TRACE(vendor_hook, handler, data, err)	\
+do {								\
+	ret = register_trace_##vendor_hook(handler, data);				\
+	if (ret) {						\
+		pr_err("sched_ext:failed to register_trace_"#vendor_hook", ret=%d\n", ret);	\
+		goto err;					\
+	}							\
+} while (0)
+
+#define UNREGISTER_TRACE(vendor_hook, handler, data)	\
+	unregister_trace_##vendor_hook(handler, data)				\
+
+extern unsigned int highres_tick_ctrl;
+extern unsigned int highres_tick_ctrl_dbg;
+
+extern int slim_walt_ctrl;
+extern int slim_walt_dump;
+extern int slim_walt_policy;
+extern int sched_ravg_window_frame_per_sec;
+extern int slim_gov_debug;
+extern int cpu7_tl;
+extern int scx_gov_ctrl;
+extern spinlock_t new_sched_ravg_window_lock;
+extern int cluster_separate;
+
+#define HMBIRD_CPUFREQ_WINDOW_ROLLOVER	BIT(31)
+#define MAX_YIELD_SLEEP		(2000000ULL)
+#define MIN_YIELD_SLEEP		(200000ULL)
+#define YIELD_DURATION		(5000ULL)
+#define DEFAULT_YIELD_SLEEP_TH	(10)
+
+struct sched_yield_state {
+	raw_spinlock_t	lock;
+	u64				last_yield_time;
+	u64				last_update_time;
+	u64				sleep_end;
+	unsigned long	yield_cnt;
+	unsigned long	yield_cnt_after_sleep;
+	unsigned long	sleep;
+	int sleep_times;
+};
+
+DECLARE_PER_CPU(struct sched_yield_state, ystate);
+
+void hmbird_window_rollover_run_once(struct rq *rq);
+void hmbird_yield_state_update_per_frame(void);
+void hmbird_misc_init(void);
+
+void hmbird_ops_init(struct hmbird_ops *hmbird_ops);
+#endif /*__HMBIRD_SCHED__*/
diff --git a/kernel/sched/hmbird/hmbird_sched_proc.c b/kernel/sched/hmbird/hmbird_sched_proc.c
new file mode 100755
index 000000000000..234768fc767a
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_sched_proc.c
@@ -0,0 +1,527 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include "hmbird_sched_proc.h"
+#include <linux/sched/hmbird_version.h>
+
+#include "hmbird_util_track.h"
+#include "slim.h"
+
+#define HMBIRD_SCHED_PROC_DIR "hmbird_sched"
+#define SLIM_FREQ_GOV_DIR       "slim_freq_gov"
+#define LOAD_TRACK_DIR          "slim_walt"
+#define HMBIRD_PROC_PERMISSION  0666
+
+int scx_enable;
+int partial_enable;
+int cpuctrl_high_ratio = 55;
+int cpuctrl_low_ratio = 40;
+int slim_stats;
+int hmbirdcore_debug;
+int slim_for_app;
+int misfit_ds = 90;
+unsigned int highres_tick_ctrl;
+unsigned int highres_tick_ctrl_dbg;
+int cpu7_tl = 70;
+int slim_walt_ctrl;
+int slim_walt_dump;
+int slim_walt_policy;
+int slim_gov_debug;
+int scx_gov_ctrl = 1;
+int sched_ravg_window_frame_per_sec = 125;
+int parctrl_high_ratio = 55;
+int parctrl_low_ratio = 40;
+int parctrl_high_ratio_l = 65;
+int parctrl_low_ratio_l = 50;
+int isoctrl_high_ratio = 75;
+int isoctrl_low_ratio = 60;
+int isolate_ctrl;
+int iso_free_rescue;
+int heartbeat;
+int heartbeat_enable = 1;
+int watchdog_enable;
+int save_gov;
+u64 cpu_cluster_masks;
+int hmbird_preempt_policy;
+int cluster_separate;
+
+char saved_gov[NR_CPUS][MAX_GOV_LEN];
+
+static int set_proc_buf_val(struct file *file, const char __user *buf, size_t count, int *val)
+{
+	char kbuf[32] = {0};
+	int err;
+
+	if (count >= 32)
+		return -EFAULT;
+
+	if (copy_from_user(kbuf, buf, count)) {
+		pr_err("hmbird_sched : Failed to copy_from_user\n");
+		return -EFAULT;
+	}
+
+	err = kstrtoint(strstrip(kbuf), 0, val);
+	if (err < 0) {
+		pr_err("hmbird_sched: Failed to exec kstrtoint\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int set_proc_buf_val_u64(struct file *file, const char __user *buf,
+				size_t count, u64 *val)
+{
+	char kbuf[32] = {0};
+	int err;
+
+	if (count >= sizeof(kbuf))
+		return -EFAULT;
+
+	if (copy_from_user(kbuf, buf, count)) {
+		pr_err("hmbird_sched : Failed to copy_from_user\n");
+		return -EFAULT;
+	}
+
+	err = kstrtou64(strstrip(kbuf), 0, val);
+	if (err < 0) {
+		pr_err("hmbird_sched: Failed to exec kstrtoul\n");
+	return -EFAULT;
+	}
+
+	return 0;
+}
+
+/* common ops begin */
+static ssize_t hmbird_common_write(struct file *file,
+				   const char __user *buf,
+				   size_t count, loff_t *ppos)
+{
+	int *pval = (int *)pde_data(file_inode(file));
+
+	if (set_proc_buf_val(file, buf, count, pval))
+		return -EFAULT;
+
+	return count;
+}
+
+static int hmbird_common_show(struct seq_file *m, void *v)
+{
+	seq_printf(m, "%d\n", *(int *) m->private);
+	return 0;
+}
+
+static int hmbird_common_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, hmbird_common_show, pde_data(inode));
+}
+
+static int hmbird_common_ul_show(struct seq_file *m, void *v)
+{
+	seq_printf(m, "%lu\n", *(unsigned long *) m->private);
+	return 0;
+}
+
+static int hmbird_common_ul_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, hmbird_common_ul_show, pde_data(inode));
+}
+
+HMBIRD_PROC_OPS(hmbird_common, hmbird_common_open, hmbird_common_write);
+/* common ops end */
+
+/* scx_enable ops begin */
+static ssize_t scx_enable_proc_write(struct file *file, const char __user *buf,
+								size_t count, loff_t *ppos)
+{
+	int *pval = (int *)pde_data(file_inode(file));
+
+	if (set_proc_buf_val(file, buf, count, pval))
+		return -EFAULT;
+
+	WRITE_ONCE(sw_type, HMBIRD_SWITCH_PROC);
+	if (hmbird_ctrl(*pval))
+		return -EFAULT;
+
+	return count;
+}
+HMBIRD_PROC_OPS(scx_enable, hmbird_common_open, scx_enable_proc_write);
+/* scx_enable ops end */
+
+/* hmbird_stats ops begin */
+#define MAX_STATS_BUF	(4096)
+static int hmbird_stats_proc_show(struct seq_file *m, void *v)
+{
+	char *buf;
+
+	buf = kmalloc(MAX_STATS_BUF, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	stats_print(buf, MAX_STATS_BUF);
+
+	seq_printf(m, "%s\n", buf);
+
+	kfree(buf);
+	return 0;
+}
+
+static int hmbird_stats_proc_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, hmbird_stats_proc_show, inode);
+}
+HMBIRD_PROC_OPS(hmbird_stats, hmbird_stats_proc_open, NULL);
+/* hmbird_stats ops end */
+
+/* sched_ravg_window_frame_per_sec ops begin */
+static ssize_t sched_ravg_window_frame_per_sec_proc_write(struct file *file,
+			const char __user *buf, size_t count, loff_t *ppos)
+{
+	int *pval = (int *)pde_data(file_inode(file));
+
+	if (set_proc_buf_val(file, buf, count, pval))
+		return -EFAULT;
+
+	sched_ravg_window_change(*pval);
+
+	return count;
+}
+HMBIRD_PROC_OPS(sched_ravg_window_frame_per_sec, hmbird_common_open,
+			sched_ravg_window_frame_per_sec_proc_write);
+/* sched_ravg_window_frame_per_sec ops end */
+
+static ssize_t save_gov_str(struct file *file, const char __user *buf,
+					size_t count, loff_t *ppos)
+{
+	int cpu;
+	struct cpufreq_policy *policy;
+
+	for_each_possible_cpu(cpu) {
+		policy = cpufreq_cpu_get(cpu);
+		if (!policy || (cpu != policy->cpu))
+			continue;
+		WARN_ON(show_scaling_governor(policy, saved_gov[cpu]) <= 0);
+		hmbird_info_systrace("<gov_restore>:save origin gov : %s\n", saved_gov[cpu]);
+	}
+	return count;
+}
+HMBIRD_PROC_OPS(save_gov, hmbird_common_open, save_gov_str);
+
+static ssize_t cpu_cluster_proc_write(struct file *file, const char __user *buf,
+							size_t count, loff_t *ppos)
+{
+	u64 *pval = (u64 *)pde_data(file_inode(file));
+
+	if (set_proc_buf_val_u64(file, buf, count, pval))
+		return -EFAULT;
+
+	if (scx_enable == 0)
+		set_cpu_cluster(*pval);
+
+	return count;
+}
+HMBIRD_PROC_OPS(cpu_cluster_masks, hmbird_common_ul_open, cpu_cluster_proc_write);
+
+static ssize_t slim_walt_ctrl_write(struct file *file, const char __user *buf,
+					size_t count, loff_t *ppos)
+{
+	int *pval = (int *)pde_data(file_inode(file));
+	int tmp_val;
+
+	if (set_proc_buf_val(file, buf, count, &tmp_val))
+		return -EFAULT;
+
+	slim_walt_enable(tmp_val);
+	*pval = tmp_val;
+
+	return count;
+}
+
+HMBIRD_PROC_OPS(slim_walt_ctrl, hmbird_common_open, slim_walt_ctrl_write);
+
+/* yield_opt ops begin */
+static int yield_opt_show(struct seq_file *m, void *v)
+{
+	struct yield_opt_params *data = m->private;
+
+	seq_printf(m, "yield_opt:{\"enable\":%d; \"frame_per_sec\":%d; \"headroom\":%d}\n",
+				data->enable, data->frame_per_sec, data->yield_headroom);
+	return 0;
+}
+
+static int yield_opt_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, yield_opt_show, pde_data(inode));
+}
+
+static ssize_t yield_opt_write(struct file *file, const char __user *buf,
+							size_t count, loff_t *ppos)
+{
+	char *data;
+	int enable_tmp, frame_per_sec_tmp, yield_headroom_tmp, cpu;
+	unsigned long flags;
+
+	data = kmalloc(count + 1, GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	if (copy_from_user(data, buf, count)) {
+		kfree(data);
+		return -EFAULT;
+	}
+
+	data[count] = '\0';
+
+	if (sscanf(data, "%d %d %d", &enable_tmp, &frame_per_sec_tmp, &yield_headroom_tmp) != 3) {
+		kfree(data);
+		return -EINVAL;
+	}
+
+	if ((enable_tmp != 0 && enable_tmp != 1) || (frame_per_sec_tmp != 30 && frame_per_sec_tmp
+			!= 60 && frame_per_sec_tmp != 90 && frame_per_sec_tmp != 120) ||
+			(yield_headroom_tmp < 1 || yield_headroom_tmp > 20)) {
+		kfree(data);
+		return -EINVAL;
+	}
+
+	yield_opt_params.frame_time_ns = NSEC_PER_SEC / frame_per_sec_tmp;
+	yield_opt_params.frame_per_sec = frame_per_sec_tmp;
+	yield_opt_params.yield_headroom = yield_headroom_tmp;
+	yield_opt_params.enable = enable_tmp;
+
+	for_each_possible_cpu(cpu) {
+		struct sched_yield_state *ys = &per_cpu(ystate, cpu);
+
+		raw_spin_lock_irqsave(&ys->lock, flags);
+		ys->last_yield_time = 0;
+		ys->last_update_time = 0;
+		ys->sleep_end = 0;
+		ys->yield_cnt = 0;
+		ys->yield_cnt_after_sleep = 0;
+		ys->sleep = 0;
+		ys->sleep_times = 0;
+		raw_spin_unlock_irqrestore(&ys->lock, flags);
+	}
+
+	kfree(data);
+	return count;
+}
+
+HMBIRD_PROC_OPS(yield_opt, yield_opt_open, yield_opt_write);
+
+
+static int __init hmbird_proc_init(void)
+{
+	struct proc_dir_entry *hmbird_dir;
+	struct proc_dir_entry *load_track_dir;
+	struct proc_dir_entry *freq_gov_dir;
+
+	if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION)
+		return 0;
+
+	/* mkdir /proc/hmbird_sched */
+	hmbird_dir = proc_mkdir(HMBIRD_SCHED_PROC_DIR, NULL);
+	if (!hmbird_dir) {
+		pr_err("Error creating proc directory %s\n", HMBIRD_SCHED_PROC_DIR);
+		return -ENOMEM;
+	}
+
+	/* /proc/hmbird_sched--begin */
+	HMBIRD_CREATE_PROC_ENTRY_DATA("scx_enable", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&scx_enable_proc_ops,
+					&scx_enable);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("partial_ctrl", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&partial_enable);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_high", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&cpuctrl_high_ratio);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("cpuctrl_low", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&cpuctrl_low_ratio);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("slim_stats", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&slim_stats);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("hmbirdcore_debug", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&hmbirdcore_debug);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("slim_for_app", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&slim_for_app);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("misfit_ds", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&misfit_ds);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("scx_shadow_tick_enable", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&highres_tick_ctrl);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("highres_tick_ctrl_dbg", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&highres_tick_ctrl_dbg);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("cpu7_tl", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&cpu7_tl);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("cpu_cluster_masks", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&cpu_cluster_masks_proc_ops,
+					&cpu_cluster_masks);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("save_gov", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&save_gov_proc_ops,
+					&save_gov);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&heartbeat);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("heartbeat_enable", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&heartbeat_enable);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("watchdog_enable", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&watchdog_enable);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("isolate_ctrl", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&isolate_ctrl);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&parctrl_high_ratio);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&parctrl_low_ratio);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_high_ratio", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&isoctrl_high_ratio);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("isoctrl_low_ratio", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&isoctrl_low_ratio);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("iso_free_rescue", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&iso_free_rescue);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_high_ratio_l", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&parctrl_high_ratio_l);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("parctrl_low_ratio_l", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&parctrl_low_ratio_l);
+
+	HMBIRD_CREATE_PROC_ENTRY("hmbird_stats", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_stats_proc_ops);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("yield_opt", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&yield_opt_proc_ops,
+					&yield_opt_params);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("hmbird_preempt_policy", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&hmbird_preempt_policy);
+	/* /proc/hmbird_sched--end */
+
+	/* mkdir /proc/hmbird_sched/slim_walt */
+	load_track_dir = proc_mkdir(LOAD_TRACK_DIR, hmbird_dir);
+	if (!load_track_dir) {
+		pr_err("Error creating proc directory %s\n", LOAD_TRACK_DIR);
+		return -ENOMEM;
+	}
+
+	/* /proc/hmbird_sched/slim_walt--begin */
+	HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_ctrl", HMBIRD_PROC_PERMISSION,
+					load_track_dir,
+					&slim_walt_ctrl_proc_ops,
+					&slim_walt_ctrl);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_dump", HMBIRD_PROC_PERMISSION,
+					load_track_dir,
+					&hmbird_common_proc_ops,
+					&slim_walt_dump);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("slim_walt_policy", HMBIRD_PROC_PERMISSION,
+					load_track_dir,
+					&hmbird_common_proc_ops,
+					&slim_walt_policy);
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("frame_per_sec", HMBIRD_PROC_PERMISSION,
+					load_track_dir,
+					&sched_ravg_window_frame_per_sec_proc_ops,
+					&sched_ravg_window_frame_per_sec);
+	/* /proc/hmbird_sched/slim_walt--end */
+
+	/* mkdir /proc/hmbird_sched/slim_freq_gov */
+	freq_gov_dir = proc_mkdir(SLIM_FREQ_GOV_DIR, hmbird_dir);
+	if (!freq_gov_dir) {
+		pr_err("Error creating proc directory %s\n", SLIM_FREQ_GOV_DIR);
+		return -ENOMEM;
+	}
+
+	/* /proc/hmbird_sched/slim_freq_gov--begin */
+	HMBIRD_CREATE_PROC_ENTRY_DATA("slim_gov_debug", HMBIRD_PROC_PERMISSION,
+					freq_gov_dir,
+					&hmbird_common_proc_ops,
+					&slim_gov_debug);
+	HMBIRD_CREATE_PROC_ENTRY_DATA("scx_gov_ctrl", HMBIRD_PROC_PERMISSION,
+					freq_gov_dir,
+					&hmbird_common_proc_ops,
+					&scx_gov_ctrl);
+	/* /proc/hmbird_sched/slim_freq_gov--end */
+
+	HMBIRD_CREATE_PROC_ENTRY_DATA("cluster_separate", HMBIRD_PROC_PERMISSION,
+					hmbird_dir,
+					&hmbird_common_proc_ops,
+					&cluster_separate);
+
+	return 0;
+}
+
+device_initcall(hmbird_proc_init);
diff --git a/kernel/sched/hmbird/hmbird_sched_proc.h b/kernel/sched/hmbird/hmbird_sched_proc.h
new file mode 100755
index 000000000000..e22bc75f1dd7
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_sched_proc.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __HMBIRD_SCHED_PROC_H__
+#define __HMBIRD_SCHED_PROC_H__
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+
+#define HMBIRD_CREATE_PROC_ENTRY(name, mode, parent, proc_ops) \
+	do { \
+		if (!proc_create(name, mode, parent, proc_ops)) { \
+			pr_err("Error creating proc entry %s\n", name); \
+			return -ENOMEM; \
+		} \
+	} while (0)
+
+#define HMBIRD_CREATE_PROC_ENTRY_DATA(name, mode, parent, proc_ops, data) \
+	do { \
+		if (!proc_create_data(name, mode, parent, proc_ops, data)) { \
+			pr_err("Error creating proc entry with data %s\n", name); \
+			return -ENOMEM; \
+		} \
+	} while (0)
+
+#define HMBIRD_PROC_OPS(name, open_func, write_func) \
+	static const struct proc_ops name##_proc_ops = { \
+		.proc_open = open_func, \
+		.proc_write = write_func, \
+		.proc_read = seq_read, \
+		.proc_lseek = seq_lseek, \
+		.proc_release = single_release, \
+	}
+
+static ssize_t hmbird_common_write(struct file *file,
+				   const char __user *buf,
+				   size_t count, loff_t *ppos);
+static int hmbird_common_show(struct seq_file *m, void *v);
+static int hmbird_common_open(struct inode *inode, struct file *file);
+
+#endif
diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.c b/kernel/sched/hmbird/hmbird_shadow_tick.c
new file mode 100755
index 000000000000..21de551c5132
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_shadow_tick.c
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+#include <linux/tick.h>
+#include "../../time/tick-sched.h"
+#include <linux/sched/hmbird_version.h>
+
+#include "hmbird_shadow_tick.h"
+
+#define HIGHRES_WATCH_CPU       0
+
+#include <linux/sched/hmbird_proc_val.h>
+static bool shadow_tick_enable(void) {return highres_tick_ctrl; }
+#ifdef CONFIG_HMBIRD_DEBUG_MODE
+static bool shadow_tick_dbg_enable(void) {return highres_tick_ctrl_dbg; }
+#endif
+static bool shadow_tick_timer_init_flag;
+
+#ifdef CONFIG_HMBIRD_DEBUG_MODE
+#define shadow_tick_printk(fmt, args...)	\
+do {							\
+	int cpu = smp_processor_id();			\
+	if (shadow_tick_dbg_enable() && cpu == HIGHRES_WATCH_CPU)	\
+		trace_printk("hmbird shadow tick :"fmt, args);	\
+} while (0)
+#else
+#define shadow_tick_printk(fmt, args...)
+#endif
+
+DEFINE_PER_CPU(struct hrtimer, stt);
+#define shadow_tick_timer(cpu) (&per_cpu(stt, (cpu)))
+#define STOP_IDLE_TRIGGER     (1)
+#define PERIODIC_TICK_TRIGGER (2)
+#define TICK_INTVAL	(1000000ULL)
+/*
+ * restart hrtimer while resume from idle. scheduler tick may resume after 4ms,
+ * so we can't restart hrtimer in scheduler tick.
+ */
+static DEFINE_PER_CPU(u8, trigger_event);
+
+/*
+ * Implement 1ms tick by inserting 3 hrtimer ticks to schduler tick.
+ * stop hrtimer when tick reachs 4, then restart it at scheduler timer handler.
+ */
+static DEFINE_PER_CPU(u8, tick_phase);
+
+static inline void highres_timer_ctrl(bool enable, int cpu)
+{
+	if (enable && hmbird_enabled()) {
+		if (!hrtimer_active(shadow_tick_timer(cpu)))
+			hrtimer_start(shadow_tick_timer(cpu),
+				ns_to_ktime(TICK_INTVAL), HRTIMER_MODE_REL_PINNED);
+	} else {
+		if (!enable)
+			hrtimer_cancel(shadow_tick_timer(cpu));
+	}
+}
+
+static inline void high_res_clear_phase(int cpu)
+{
+	per_cpu(tick_phase, cpu) = 0;
+}
+
+static enum hrtimer_restart highres_next_phase(int cpu, struct hrtimer *timer)
+{
+	per_cpu(tick_phase, cpu) = ++per_cpu(tick_phase, cpu) % 3;
+	if (per_cpu(tick_phase, cpu)) {
+		hrtimer_forward_now(timer, ns_to_ktime(TICK_INTVAL));
+		return HRTIMER_RESTART;
+	}
+	return HRTIMER_NORESTART;
+}
+
+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev,
+		struct task_struct *next, unsigned int prev_state)
+{
+	int cpu = smp_processor_id();
+
+	if (shadow_tick_enable() && (cpu_rq(cpu)->idle == prev)) {
+		per_cpu(trigger_event, cpu) = STOP_IDLE_TRIGGER;
+		high_res_clear_phase(cpu);
+		highres_timer_ctrl(true, cpu);
+	}
+}
+
+static enum hrtimer_restart scheduler_tick_no_balance(struct hrtimer *timer)
+{
+	int cpu = smp_processor_id();
+	struct rq *rq = cpu_rq(cpu);
+	struct task_struct *curr = rq->curr;
+	struct rq_flags rf;
+
+	rq_lock(rq, &rf);
+	update_rq_clock(rq);
+	curr->sched_class->task_tick(rq, curr, 0);
+	rq_unlock(rq, &rf);
+
+	return highres_next_phase(cpu, timer);
+}
+
+void shadow_tick_timer_init(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		hrtimer_init(shadow_tick_timer(cpu), CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+		shadow_tick_timer(cpu)->function = &scheduler_tick_no_balance;
+	}
+}
+
+void start_shadow_tick_timer(void)
+{
+	int cpu = smp_processor_id();
+
+	if (shadow_tick_enable()) {
+		if (per_cpu(trigger_event, cpu) == STOP_IDLE_TRIGGER)
+			highres_timer_ctrl(false, cpu);
+		per_cpu(trigger_event, cpu) = PERIODIC_TICK_TRIGGER;
+		high_res_clear_phase(cpu);
+		highres_timer_ctrl(true, cpu);
+	}
+}
+
+static void stop_shadow_tick_timer(void)
+{
+	int cpu = smp_processor_id();
+
+	per_cpu(trigger_event, cpu) = 0;
+	high_res_clear_phase(cpu);
+	highres_timer_ctrl(false, cpu);
+}
+
+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data)
+{
+	if (!shadow_tick_timer_init_flag)
+		return;
+	stop_shadow_tick_timer();
+}
+
+void scheduler_tick_handler(void *unused, struct rq *rq)
+{
+	if (!shadow_tick_timer_init_flag)
+		return;
+	start_shadow_tick_timer();
+}
+
+static int __init hmbird_shadow_tick_init(void)
+{
+	int ret = 0;
+
+	if (get_hmbird_version_type() != HMBIRD_OGKI_VERSION)
+		return 0;
+	shadow_tick_timer_init();
+	shadow_tick_timer_init_flag = true;
+	return ret;
+}
+
+device_initcall(hmbird_shadow_tick_init);
diff --git a/kernel/sched/hmbird/hmbird_shadow_tick.h b/kernel/sched/hmbird/hmbird_shadow_tick.h
new file mode 100755
index 000000000000..0c26fd984f0a
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_shadow_tick.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __HMBIRD_SHADOW_TICK_H__
+#define __HMBIRD_SHADOW_TICK_H__
+
+#include <linux/sched.h>
+
+void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data);
+void scheduler_tick_handler(void *unused, struct rq *rq);
+void sched_switch_handler(void *data, bool preempt, struct task_struct *prev,
+		struct task_struct *next, unsigned int prev_state);
+#endif
diff --git a/kernel/sched/hmbird/hmbird_trace.h b/kernel/sched/hmbird/hmbird_trace.h
new file mode 100755
index 000000000000..8468b406f347
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_trace.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM hmbird
+
+#if !defined(_TRACE_HMBIRD_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_HMBIRD_H
+
+#include <linux/sched.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#define MAX_FATAL_INFO (64)
+
+TRACE_EVENT(hmbird_fatal_info,
+
+	TP_PROTO(unsigned int type, int pe, int lnr, int bnr, char *info),
+
+	TP_ARGS(type, pe, lnr, bnr, info),
+
+	TP_STRUCT__entry(
+		__field(unsigned int, type)
+		__field(int, pe)
+		__field(int, lnr)
+		__field(int, bnr)
+		__array(char, info, MAX_FATAL_INFO)),
+
+	TP_fast_assign(
+		__entry->type = type;
+		__entry->pe = pe;
+		__entry->lnr = lnr;
+		__entry->bnr = bnr;
+		memcpy(__entry->info, info, MAX_FATAL_INFO);),
+
+	TP_printk("hmbird fatal error type=%u pe=%d lnr=%d bnr=%d info=%s",
+		__entry->type, __entry->pe, __entry->lnr, __entry->bnr,
+		__entry->info)
+);
+
+TRACE_EVENT(hmbird_update_history,
+
+	TP_PROTO(struct hmbird_entity *hmbird, struct rq *rq,
+		struct task_struct *p, u32 runtime, int samples, int event),
+
+	TP_ARGS(hmbird, rq, p, runtime, samples, event),
+
+	TP_STRUCT__entry(
+		__array(char,			comm, TASK_COMM_LEN)
+		__field(pid_t,			pid)
+		__field(unsigned int,		runtime)
+		__field(int,			samples)
+		__field(int,	event)
+		__field(unsigned int,		demand)
+		__array(u32,			hist, RAVG_HIST_SIZE)
+		__field(u16,			task_util)
+		__field(int,			cpu)),
+
+	TP_fast_assign(
+		memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
+		__entry->pid		= p->pid;
+		__entry->runtime	= runtime;
+		__entry->samples	= samples;
+		__entry->event		= event;
+		__entry->demand		= hmbird->sts.demand;
+		memcpy(__entry->hist, hmbird->sts.sum_history,
+					RAVG_HIST_SIZE * sizeof(u32));
+		__entry->task_util	= hmbird->sts.demand_scaled;
+		__entry->cpu		= rq->cpu;),
+
+	TP_printk("comm=%s[%d]: runtime %u samples %d event %d demand %u (hist: %u %u %u %u %u) task_util %u cpu %d",
+		__entry->comm, __entry->pid,
+		__entry->runtime, __entry->samples,
+		__entry->event,
+		__entry->demand,
+		__entry->hist[0], __entry->hist[1],
+		__entry->hist[2], __entry->hist[3],
+		__entry->hist[4],
+		__entry->task_util,
+		__entry->cpu)
+);
+
+#endif /*_TRACE_HMBIRD_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../../kernel/sched/hmbird
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE hmbird_trace
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/kernel/sched/hmbird/hmbird_util_track.c b/kernel/sched/hmbird/hmbird_util_track.c
new file mode 100755
index 000000000000..d520a8403401
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_util_track.c
@@ -0,0 +1,626 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 Oplus. All rights reserved.
+ */
+#include "hmbird_sched.h"
+#include "hmbird_util_track.h"
+
+#define CREATE_TRACE_POINTS
+#include "hmbird_trace.h"
+#undef CREATE_TRACE_POINTS
+
+#define HMBIRD_DEBUG_PANIC		(1 << 3)
+static bool init_irq_work_inited;
+
+extern noinline int tracing_mark_write(const char *buf);
+
+int hmbird_sched_ravg_window = 8000000;
+int new_hmbird_sched_ravg_window = 8000000;
+DEFINE_SPINLOCK(new_sched_ravg_window_lock);
+DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats);
+
+
+static struct irq_work hmbird_slim_walt_irq_work;
+
+/*Sysctl related interface*/
+#define WINDOW_STATS_RECENT		0
+#define WINDOW_STATS_MAX		1
+#define WINDOW_STATS_MAX_RECENT_AVG	2
+#define WINDOW_STATS_AVG		3
+#define WINDOW_STATS_INVALID_POLICY	4
+
+
+#define SCX_SCHED_CAPACITY_SHIFT  10
+#define SCHED_ACCOUNT_WAIT_TIME 0
+
+atomic64_t hmbird_irq_work_lastq_ws;
+static u64 tick_sched_clock;
+
+static inline u64 scale_exec_time(u64 delta, struct rq *rq)
+{
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+
+	return (delta * srq->task_exec_scale) >> SCX_SCHED_CAPACITY_SHIFT;
+}
+
+static inline u64 scale_time_to_util(u64 d)
+{
+	do_div(d, hmbird_sched_ravg_window >> SCX_SCHED_CAPACITY_SHIFT);
+	return d;
+}
+
+static u64 add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta)
+{
+	struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts);
+
+	delta = scale_exec_time(delta, rq);
+	sts->sum += delta;
+	if (unlikely(sts->sum > hmbird_sched_ravg_window))
+		sts->sum = hmbird_sched_ravg_window;
+
+	return delta;
+}
+
+
+static int
+account_busy_for_task_demand(struct rq *rq, struct task_struct *p, int event)
+{
+	/*
+	 * No need to bother updating task demand for the idle task.
+	 */
+	if (is_idle_task(p))
+		return 0;
+
+	/*
+	 * When a task is waking up it is completing a segment of non-busy
+	 * time. Likewise, if wait time is not treated as busy time, then
+	 * when a task begins to run or is migrated, it is not running and
+	 * is completing a segment of non-busy time.
+	 */
+	if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME &&
+			 (event == PICK_NEXT_TASK || event == TASK_MIGRATE)))
+		return 0;
+
+	/*
+	 * The idle exit time is not accounted for the first task _picked_ up to
+	 * run on the idle CPU.
+	 */
+	if (event == PICK_NEXT_TASK && rq->curr == rq->idle)
+		return 0;
+
+	/*
+	 * TASK_UPDATE can be called on sleeping task, when its moved between
+	 * related groups
+	 */
+	if (event == TASK_UPDATE) {
+		if (rq->curr == p)
+			return 1;
+
+		return p->on_rq ? SCHED_ACCOUNT_WAIT_TIME : 0;
+	}
+
+	return 1;
+}
+
+static void rollover_cpu_window(struct rq *rq, bool full_window)
+{
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+	u64 curr_sum = srq->curr_runnable_sum;
+
+	if (unlikely(full_window))
+		curr_sum = 0;
+
+	srq->prev_runnable_sum = curr_sum;
+	srq->curr_runnable_sum = 0;
+}
+
+static u64
+update_window_start(struct rq *rq, u64 wallclock, int event)
+{
+	s64 delta;
+	int nr_windows;
+	bool full_window;
+
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+	u64 old_window_start = srq->window_start;
+
+	if (wallclock < srq->latest_clock)
+		wallclock = srq->latest_clock;
+	delta = wallclock - srq->window_start;
+	if (delta < 0) {
+		delta = 0;
+		wallclock = srq->window_start;
+	}
+	srq->latest_clock = wallclock;
+	if (delta < hmbird_sched_ravg_window)
+		return old_window_start;
+
+	nr_windows = div64_u64(delta, hmbird_sched_ravg_window);
+	srq->window_start += (u64)nr_windows * (u64)hmbird_sched_ravg_window;
+
+	srq->prev_window_size = hmbird_sched_ravg_window;
+	full_window = nr_windows > 1;
+	rollover_cpu_window(rq, full_window);
+
+	return old_window_start;
+}
+
+#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y)
+
+static inline unsigned int get_max_freq(unsigned int cpu)
+{
+	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+
+	return (policy == NULL) ? 0 : policy->cpuinfo.max_freq;
+}
+
+static inline unsigned int cpu_cur_freq(int cpu)
+{
+	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+
+	return (policy == NULL) ? 0 : policy->cur;
+}
+
+static void
+update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,
+			  u64 wallclock)
+{
+	int cpu = cpu_of(rq);
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+
+	srq->task_exec_scale = DIV64_U64_ROUNDUP(cpu_cur_freq(cpu) *
+					arch_scale_cpu_capacity(cpu), get_max_freq(cpu));
+}
+
+/*
+ * Called when new window is starting for a task, to record cpu usage over
+ * recently concluded window(s). Normally 'samples' should be 1. It can be > 1
+ * when, say, a real-time task runs without preemption for several windows at a
+ * stretch.
+ */
+static void update_history(struct rq *rq, struct task_struct *p,
+			 u32 runtime, int samples, int event)
+{
+	struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts);
+	u32 *hist = &sts->sum_history[0];
+	int i;
+	u32 max = 0, avg, demand;
+	u64 sum = 0;
+	u16 demand_scaled;
+
+	/* Ignore windows where task had no activity */
+	if (!runtime || is_idle_task(p) || !samples)
+		goto done;
+
+	/* Push new 'runtime' value onto stack */
+	for (; samples > 0; samples--) {
+		hist[sts->cidx] = runtime;
+		sts->cidx = ++(sts->cidx) % RAVG_HIST_SIZE;
+	}
+
+	for (i = 0; i < RAVG_HIST_SIZE; i++) {
+		sum += hist[i];
+		if (hist[i] > max)
+			max = hist[i];
+	}
+
+	sts->sum = 0;
+	avg = div64_u64(sum, RAVG_HIST_SIZE);
+
+	switch (slim_walt_policy) {
+	case WINDOW_STATS_RECENT:
+		demand = runtime;
+		break;
+	case WINDOW_STATS_MAX:
+		demand = max;
+		break;
+	case WINDOW_STATS_AVG:
+		demand = avg;
+		break;
+	default:
+		demand = max(avg, runtime);
+	}
+
+	demand_scaled = scale_time_to_util(demand);
+
+	sts->demand = demand;
+	sts->demand_scaled = demand_scaled;
+
+done:
+	return;
+}
+
+
+static u64 update_task_demand(struct task_struct *p, struct rq *rq,
+			       int event, u64 wallclock)
+{
+	struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts);
+
+	u64 mark_start = sts->mark_start;
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+
+
+	u64 delta, window_start = srq->window_start;
+	int new_window, nr_full_windows;
+	u32 window_size = hmbird_sched_ravg_window;
+	u64 runtime;
+
+	new_window = mark_start < window_start;
+	if (!account_busy_for_task_demand(rq, p, event)) {
+		if (new_window)
+			/*
+			 * If the time accounted isn't being accounted as
+			 * busy time, and a new window started, only the
+			 * previous window need be closed out with the
+			 * pre-existing demand. Multiple windows may have
+			 * elapsed, but since empty windows are dropped,
+			 * it is not necessary to account those.
+			 */
+			update_history(rq, p, sts->sum, 1, event);
+		return 0;
+	}
+
+	if (!new_window) {
+		/*
+		 * The simple case - busy time contained within the existing
+		 * window.
+		 */
+		return add_to_task_demand(rq, p, wallclock - mark_start);
+	}
+
+	/*
+	 * Busy time spans at least two windows. Temporarily rewind
+	 * window_start to first window boundary after mark_start.
+	 */
+	delta = window_start - mark_start;
+	nr_full_windows = div64_u64(delta, window_size);
+	window_start -= (u64)nr_full_windows * (u64)window_size;
+
+	/* Process (window_start - mark_start) first */
+	runtime = add_to_task_demand(rq, p, window_start - mark_start);
+
+	/* Push new sample(s) into task's demand history */
+	update_history(rq, p, sts->sum, 1, event);
+	if (nr_full_windows) {
+		u64 scaled_window = scale_exec_time(window_size, rq);
+
+		update_history(rq, p, scaled_window, nr_full_windows, event);
+		runtime += nr_full_windows * scaled_window;
+	}
+
+	/*
+	 * Roll window_start back to current to process any remainder
+	 * in current window.
+	 */
+	window_start += (u64)nr_full_windows * (u64)window_size;
+
+	/* Process (wallclock - window_start) next */
+	mark_start = window_start;
+	runtime += add_to_task_demand(rq, p, wallclock - mark_start);
+
+	return runtime;
+}
+
+u16 slim_walt_cpu_util(int cpu)
+{
+	u64 prev_runnable_sum;
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu);
+
+	prev_runnable_sum = srq->prev_runnable_sum;
+	do_div(prev_runnable_sum, srq->prev_window_size >> SCX_SCHED_CAPACITY_SHIFT);
+
+	return (u16)prev_runnable_sum;
+}
+
+static DEFINE_PER_CPU(u16, prev_cpu_util);
+static inline void cpu_util_update_systrace_c(int cpu)
+{
+	char buf[256];
+	u16 cpu_util = slim_walt_cpu_util(cpu);
+
+	if (cpu_util != per_cpu(prev_cpu_util, cpu)) {
+		snprintf(buf, sizeof(buf), "C|9999|Cpu%d_util|%u\n",
+						cpu, cpu_util);
+		tracing_mark_write(buf);
+		per_cpu(prev_cpu_util, cpu) = cpu_util;
+	}
+}
+
+
+static inline int account_busy_for_cpu_time(struct rq *rq,
+					    struct task_struct *p,
+					    int event)
+{
+	return !is_idle_task(p) && (event == PUT_PREV_TASK
+					|| event == TASK_UPDATE);
+}
+
+
+static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
+				 int event, u64 wallclock)
+{
+	int new_window, full_window = 0;
+	struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts);
+	u64 mark_start = sts->mark_start;
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+	u64 window_start = srq->window_start;
+	u32 window_size = srq->prev_window_size;
+	u64 delta;
+	u64 *curr_runnable_sum = &srq->curr_runnable_sum;
+	u64 *prev_runnable_sum = &srq->prev_runnable_sum;
+
+	new_window = mark_start < window_start;
+	if (new_window)
+		full_window = (window_start - mark_start) >= window_size;
+
+
+	if (!account_busy_for_cpu_time(rq, p, event))
+		goto done;
+
+
+	if (!new_window) {
+		/*
+		 * account_busy_for_cpu_time() = 1 so busy time needs
+		 * to be accounted to the current window. No rollover
+		 * since we didn't start a new window. An example of this is
+		 * when a task starts execution and then sleeps within the
+		 * same window.
+		 */
+		delta = wallclock - mark_start;
+
+		delta = scale_exec_time(delta, rq);
+		*curr_runnable_sum += delta;
+
+		goto done;
+	}
+
+	/*
+	 * situations below this need window rollover,
+	 * Rollover of cpu counters (curr/prev_runnable_sum) should have already be done
+	 * in update_window_start()
+	 *
+	 * For task counters curr/prev_window[_cpu] are rolled over in the early part of
+	 * this function. If full_window(s) have expired and time since last update needs
+	 * to be accounted as busy time, set the prev to a complete window size time, else
+	 * add the prev window portion.
+	 *
+	 * For task curr counters a new window has begun, always assign
+	 */
+
+	/*
+	 * account_busy_for_cpu_time() = 1 so busy time needs
+	 * to be accounted to the current window. A new window
+	 * must have been started in udpate_window_start()
+	 * If any of these three above conditions are true
+	 * then this busy time can't be accounted as irqtime.
+	 *
+	 * Busy time for the idle task need not be accounted.
+	 *
+	 * An example of this would be a task that starts execution
+	 * and then sleeps once a new window has begun.
+	 */
+
+	/*
+	 * A full window hasn't elapsed, account partial
+	 * contribution to previous completed window.
+	 */
+
+
+	delta = full_window ? scale_exec_time(window_size, rq) :
+				scale_exec_time(window_start - mark_start, rq);
+
+	*prev_runnable_sum += delta;
+
+	/* Account piece of busy time in the current window. */
+	delta = scale_exec_time(wallclock - window_start, rq);
+	*curr_runnable_sum += delta;
+
+done:
+	if (slim_walt_dump && new_window)
+		cpu_util_update_systrace_c(rq->cpu);
+}
+
+
+void slim_walt_window_rollover_run_once(u64 old_window_start, struct rq *rq)
+{
+	u64 result;
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+	u64 new_window_start = srq->window_start;
+
+	if (old_window_start == new_window_start)
+		return;
+
+	result = atomic64_cmpxchg(&hmbird_irq_work_lastq_ws,
+					old_window_start, new_window_start);
+	if (result != old_window_start)
+		return;
+
+	if (likely(cpu_online(raw_smp_processor_id())))
+		irq_work_queue(&hmbird_slim_walt_irq_work);
+	else
+		irq_work_queue_on(&hmbird_slim_walt_irq_work, cpumask_any(cpu_online_mask));
+}
+
+/*
+ * In the core scheduler, most of the load update points update the rq_clock after
+ * holding the rq lock. We can directly use rq_clock to reduce the overhead of
+ * obtaining the time, but to prevent the subsequent migration of the load update
+ * point to before the update of rq_clock, we wrap the judgment of the rq_clock
+ * update here.
+ */
+void hmbird_update_task_ravg_rqclock_wrapper(struct task_struct *p,
+				struct rq *rq, int event)
+{
+	if (!(rq->clock_update_flags & RQCF_UPDATED))
+		update_rq_clock(rq);
+
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+
+	hmbird_update_task_ravg(p, rq, event, max(rq_clock(rq), srq->latest_clock));
+}
+
+void hmbird_update_task_ravg(struct task_struct *p,
+				struct rq *rq, int event, u64 wallclock)
+{
+	struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts);
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+	u64 old_window_start;
+
+	if (!slim_walt_ctrl)
+		return;
+
+	if (!srq->window_start || sts->mark_start == wallclock)
+		return;
+
+	old_window_start = update_window_start(rq, wallclock, event);
+
+	if (!sts->window_start)
+		sts->window_start = srq->window_start;
+
+	if (!sts->mark_start)
+		goto done;
+
+	update_task_rq_cpu_cycles(p, rq, event, wallclock);
+	update_task_demand(p, rq, event, wallclock);
+	update_cpu_busy_time(p, rq, event, wallclock);
+
+	sts->window_start = srq->window_start;
+
+done:
+	sts->mark_start = wallclock;
+	slim_walt_window_rollover_run_once(old_window_start, rq);
+}
+
+static void slim_walt_irq_work(struct irq_work *irq_work)
+{
+	cpumask_t lock_cpus;
+	struct hmbird_sched_rq_stats *srq;
+	struct rq *rq;
+	int cpu;
+	int level = 0;
+	u64 wc;
+	unsigned long flags;
+
+	cpumask_copy(&lock_cpus, cpu_possible_mask);
+
+	for_each_cpu(cpu, &lock_cpus) {
+		if (level == 0)
+			raw_spin_lock(&cpu_rq(cpu)->__lock);
+		else
+			raw_spin_lock_nested(&cpu_rq(cpu)->__lock, level);
+		level++;
+	}
+
+	wc = sched_clock();
+
+	for_each_cpu(cpu, &lock_cpus) {
+		rq = cpu_rq(cpu);
+		hmbird_update_task_ravg(rq->curr, rq, TASK_UPDATE, wc);
+	}
+
+	cpufreq_update_util(cpu_rq(0), HMBIRD_CPUFREQ_WINDOW_ROLLOVER);
+	spin_lock_irqsave(&new_sched_ravg_window_lock, flags);
+	if (unlikely(new_hmbird_sched_ravg_window != hmbird_sched_ravg_window)) {
+		srq = &per_cpu(hmbird_sched_rq_stats, smp_processor_id());
+		if (wc < srq->window_start + new_hmbird_sched_ravg_window)
+			hmbird_sched_ravg_window = new_hmbird_sched_ravg_window;
+	}
+	spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags);
+
+	for_each_cpu(cpu, &lock_cpus) {
+		raw_spin_unlock(&cpu_rq(cpu)->__lock);
+	}
+}
+static void hmbird_sched_init_rq(struct rq *rq)
+{
+	struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu_of(rq));
+
+	srq->prev_window_size = hmbird_sched_ravg_window;
+	srq->task_exec_scale = 1024;
+	srq->window_start = 0;
+}
+
+void hmbird_sched_init_task(struct task_struct *p)
+{
+	struct hmbird_sched_task_stats *sts = &(get_hmbird_ts(p)->sts);
+
+	memset(sts, 0, sizeof(struct hmbird_sched_task_stats));
+}
+
+static void hmbird_sched_stats_init(void)
+{
+	unsigned long flags;
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		raw_spin_lock_irqsave(&rq->__lock, flags);
+		hmbird_sched_init_rq(rq);
+		raw_spin_unlock_irqrestore(&rq->__lock, flags);
+	}
+	slim_walt_policy = WINDOW_STATS_MAX_RECENT_AVG;
+
+	if (false == init_irq_work_inited) {
+		init_irq_work(&hmbird_slim_walt_irq_work, slim_walt_irq_work);
+		init_irq_work_inited = true;
+	}
+}
+
+void hmbird_scheduler_tick(void)
+{
+	int cpu = smp_processor_id();
+	struct rq *rq = cpu_rq(cpu);
+
+	if (unlikely(!tick_sched_clock)) {
+		/*
+		 * Let the window begin 20us prior to the tick,
+		 * that way we are guaranteed a rollover when the tick occurs.
+		 * Use rq->clock directly instead of rq_clock() since
+		 * we do not have the rq lock and
+		 * rq->clock was updated in the tick callpath.
+		 */
+		if (cmpxchg64(&tick_sched_clock, 0, rq->clock - 20000))
+			return;
+		for_each_possible_cpu(cpu) {
+			struct hmbird_sched_rq_stats *srq = &per_cpu(hmbird_sched_rq_stats, cpu);
+
+			srq->window_start = tick_sched_clock;
+		}
+		atomic64_set(&hmbird_irq_work_lastq_ws, tick_sched_clock);
+	}
+}
+
+void slim_walt_enable(int enable)
+{
+	if (1 == !!enable) {
+		hmbird_sched_stats_init();
+		WRITE_ONCE(tick_sched_clock, 0);
+	} else
+		slim_walt_ctrl = 0;
+}
+
+void slim_get_cpu_util(int cpu, u64 *util)
+{
+	if (cpu < 0)
+		return;
+
+	*util = slim_walt_cpu_util(cpu);
+}
+
+void slim_get_task_util(struct task_struct *p, u64 *util)
+{
+	if (p == NULL)
+		return;
+
+	*util = get_hmbird_ts(p)->sts.demand_scaled;
+}
+
+void sched_ravg_window_change(int frame_per_sec)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&new_sched_ravg_window_lock, flags);
+	new_hmbird_sched_ravg_window = NSEC_PER_SEC / frame_per_sec;
+	spin_unlock_irqrestore(&new_sched_ravg_window_lock, flags);
+}
diff --git a/kernel/sched/hmbird/hmbird_util_track.h b/kernel/sched/hmbird/hmbird_util_track.h
new file mode 100755
index 000000000000..17b85df7f83b
--- /dev/null
+++ b/kernel/sched/hmbird/hmbird_util_track.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * HMBIRD scheduler class
+ *
+ * Copyright (c) 2024 OPlus.
+ * Copyright (c) 2024 Dao Huang
+ * Copyright (c) 2024 Yuxing Wang
+ * Copyright (c) 2024 Taiyu Li
+ */
+#ifndef __HMBIRD_UTIL_TRACK_H__
+#define __HMBIRD_UTIL_TRACK_H__
+
+void hmbird_update_task_ravg(struct task_struct *p,
+				struct rq *rq, int event, u64 wallclock);
+void hmbird_sched_init_task(struct task_struct *p);
+void slim_walt_enable(int enable);
+void slim_get_cpu_util(int cpu, u64 *util);
+void slim_get_task_util(struct task_struct *p, u64 *util);
+
+extern atomic64_t hmbird_irq_work_lastq_ws;
+
+enum task_event {
+	PUT_PREV_TASK   = 0,
+	PICK_NEXT_TASK  = 1,
+	TASK_WAKE       = 2,
+	TASK_MIGRATE    = 3,
+	TASK_UPDATE     = 4,
+	IRQ_UPDATE      = 5,
+};
+
+extern DEFINE_PER_CPU(struct hmbird_sched_rq_stats, hmbird_sched_rq_stats);
+
+#endif /* __HMBIRD_UTIL_TRACK_H__ */
diff --git a/kernel/sched/hmbird/slim.h b/kernel/sched/hmbird/slim.h
new file mode 100755
index 000000000000..eb386260e950
--- /dev/null
+++ b/kernel/sched/hmbird/slim.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __SLIM_H
+#define __SLIM_H
+
+extern atomic_t __hmbird_ops_enabled;
+extern atomic_t non_hmbird_task;
+extern int cgroup_ids_table[NUMS_CGROUP_KINDS];
+extern int heartbeat;
+extern int heartbeat_enable;
+extern int watchdog_enable;
+extern int isolate_ctrl;
+extern int parctrl_high_ratio;
+extern int parctrl_low_ratio;
+extern int isoctrl_high_ratio;
+extern int isoctrl_low_ratio;
+extern int iso_free_rescue;
+extern int yield_opt;
+
+extern enum hmbird_switch_type sw_type;
+extern noinline int tracing_mark_write(const char *buf);
+int task_top_id(struct task_struct *p);
+void stats_print(char *buf, int len);
+void hmbird_skip_yield(long *skip);
+extern spinlock_t hmbird_tasks_lock;
+
+struct yield_opt_params {
+	int enable;
+	int frame_per_sec;
+	u64 frame_time_ns;
+	int yield_headroom;
+};
+
+extern struct yield_opt_params yield_opt_params;
+
+#define MAX_GOV_LEN     (16)
+extern char saved_gov[NR_CPUS][MAX_GOV_LEN];
+
+#endif
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index b33cefeb4188..487255ac6832 100755
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -408,13 +408,17 @@ static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int fl
 
 static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
 {
-	scx_update_idle(rq, false);
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_update_idle(rq, false);
+#endif
 }
 
 static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)
 {
 	update_idle_core(rq);
-	scx_update_idle(rq, true);
+#ifdef CONFIG_HMBIRD_SCHED
+	hmbird_update_idle(rq, true);
+#endif
 	schedstat_inc(rq->sched_goidle);
 }
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 509e8dc616ab..ab546ffc244b 100755
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -188,18 +188,13 @@ static inline int idle_policy(int policy)
 	return policy == SCHED_IDLE;
 }
 
-static inline int normal_policy(int policy)
-{
-#ifdef CONFIG_SCHED_CLASS_EXT
-	if (policy == SCHED_EXT)
-		return true;
-#endif
-	return policy == SCHED_NORMAL;
-}
-
 static inline int fair_policy(int policy)
 {
-	return normal_policy(policy) || policy == SCHED_BATCH;
+#ifdef CONFIG_HMBIRD_SCHED
+	return policy == SCHED_HMBIRD || policy == SCHED_NORMAL || policy == SCHED_BATCH;
+#else
+	return policy == SCHED_NORMAL || policy == SCHED_BATCH;
+#endif
 }
 
 static inline int rt_policy(int policy)
@@ -247,24 +242,6 @@ static inline void update_avg(u64 *avg, u64 sample)
 #define shr_bound(val, shift)							\
 	(val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1))
 
-/*
- * cgroup weight knobs should use the common MIN, DFL and MAX values which are
- * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it
- * maps pretty well onto the shares value used by scheduler and the round-trip
- * conversions preserve the original value over the entire range.
- */
-static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight)
-{
-	return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL);
-}
-
-static inline unsigned long sched_weight_to_cgroup(unsigned long weight)
-{
-	return clamp_t(unsigned long,
-		       DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024),
-		       CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX);
-}
-
 /*
  * !! For sched_setattr_nocheck() (kernel) only !!
  *
@@ -532,10 +509,12 @@ static inline void set_task_rq_fair(struct sched_entity *se,
 			     struct cfs_rq *prev, struct cfs_rq *next) { }
 #endif /* CONFIG_SMP */
 #else /* CONFIG_FAIR_GROUP_SCHED */
+#ifdef CONFIG_HMBIRD_SCHED
 static inline int sched_group_set_shares(struct task_group *tg, unsigned long shares)
 {
 	return 0;
 }
+#endif
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #else /* CONFIG_CGROUP_SCHED */
@@ -700,29 +679,6 @@ struct cfs_rq {
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 };
 
-#ifdef CONFIG_SCHED_CLASS_EXT
-/* scx_rq->flags, protected by the rq lock */
-enum scx_rq_flags {
-	SCX_RQ_CAN_STOP_TICK	= 1 << 0,
-};
-
-struct scx_rq {
-	struct scx_dispatch_q	local_dsq;
-	struct list_head	watchdog_list;
-	u64			ops_qseq;
-	u64			extra_enq_flags;	/* see move_task_to_local_dsq() */
-	u32			nr_running;
-	u32			flags;
-	bool			cpu_released;
-	cpumask_var_t		cpus_to_kick;
-	cpumask_var_t		cpus_to_preempt;
-	cpumask_var_t		cpus_to_wait;
-	u64			pnt_seq;
-	struct irq_work		kick_cpus_irq_work;
-	struct rq               *rq;
-};
-#endif /* CONFIG_SCHED_CLASS_EXT */
-
 static inline int rt_bandwidth_enabled(void)
 {
 	return sysctl_sched_rt_runtime >= 0;
@@ -1258,11 +1214,7 @@ struct rq {
 
 	ANDROID_OEM_DATA_ARRAY(1, 16);
 
-#ifdef CONFIG_SLIM_SCHED
-	ANDROID_KABI_USE(1, struct scx_rq *scx);
-#else
 	ANDROID_KABI_RESERVE(1);
-#endif
 	ANDROID_KABI_RESERVE(2);
 	ANDROID_KABI_RESERVE(3);
 	ANDROID_KABI_RESERVE(4);
@@ -2340,11 +2292,6 @@ struct affinity_context {
 	unsigned int flags;
 };
 
-enum rq_onoff_reason {
-	RQ_ONOFF_HOTPLUG,               /* CPU is going on/offline */
-	RQ_ONOFF_TOPOLOGY,              /* sched domain topology update */
-};
-
 struct sched_class {
 
 #ifdef CONFIG_UCLAMP_TASK
@@ -2551,7 +2498,6 @@ extern void init_sched_rt_class(void);
 extern void init_sched_fair_class(void);
 
 extern void reweight_task(struct task_struct *p, int prio);
-extern void __setscheduler_prio(struct task_struct *p, int prio);
 
 extern void resched_curr(struct rq *rq);
 extern void resched_cpu(int cpu);
@@ -2632,53 +2578,6 @@ static inline void sub_nr_running(struct rq *rq, unsigned count)
 extern void activate_task(struct rq *rq, struct task_struct *p, int flags);
 extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
 
-struct sched_change_guard {
-	struct task_struct	*p;
-	struct rq		*rq;
-	bool			queued;
-	bool			running;
-	bool			done;
-};
-
-extern struct sched_change_guard
-sched_change_guard_init(struct rq *rq, struct task_struct *p, int flags);
-
-extern void sched_change_guard_fini(struct sched_change_guard *cg, int flags);
-
-/**
- * SCHED_CHANGE_BLOCK - Nested block for task attribute updates
- * @__rq: Runqueue the target task belongs to
- * @__p: Target task
- * @__flags: DEQUEUE/ENQUEUE_* flags
- *
- * A task may need to be dequeued and put_prev_task'd for attribute updates and
- * set_next_task'd and re-enqueued afterwards. This helper defines a nested
- * block which automatically handles these preparation and cleanup operations.
- *
- *  SCHED_CHANGE_BLOCK(rq, p, DEQUEUE_SAVE | DEQUEUE_NOCLOCK) {
- *	  update_attribute(p);
- *        ...
- *  }
- *
- * If @__flags is a variable, the variable may be updated in the block body and
- * the updated value will be used when re-enqueueing @p.
- *
- * If %DEQUEUE_NOCLOCK is specified, the caller is responsible for calling
- * update_rq_clock() beforehand. Otherwise, the rq clock is automatically
- * updated iff the task needs to be dequeued and re-enqueued. Only the former
- * case guarantees that the rq clock is up-to-date inside and after the block.
- */
-#define SCHED_CHANGE_BLOCK(__rq, __p, __flags)					\
-	for (struct sched_change_guard __cg =					\
-			sched_change_guard_init(__rq, __p, __flags);		\
-	     !__cg.done; sched_change_guard_fini(&__cg, __flags))
-
-extern void check_class_changing(struct rq *rq, struct task_struct *p,
-				 const struct sched_class *prev_class);
-extern void check_class_changed(struct rq *rq, struct task_struct *p,
-				const struct sched_class *prev_class,
-				int oldprio);
-
 extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags);
 
 #ifdef CONFIG_PREEMPT_RT
@@ -3710,6 +3609,8 @@ static inline bool cpu_busy_with_softirqs(int cpu)
 }
 #endif /* CONFIG_RT_SOFTIRQ_AWARE_SCHED */
 
-#include "ext.h"
+#ifdef CONFIG_HMBIRD_SCHED
+#include "hmbird.h"
+#endif
 
 #endif /* _KERNEL_SCHED_SCHED_H */
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 998c2eefe173..2a3f819b9e88 100755
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -34,6 +34,10 @@
 
 #include <trace/events/timer.h>
 
+#ifdef CONFIG_HMBIRD_SCHED
+#include "../sched/hmbird/hmbird_shadow_tick.h"
+#endif
+
 /*
  * Per-CPU nohz control structure
  */
@@ -1112,6 +1116,9 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 	return true;
 }
 
+#ifdef CONFIG_HMBIRD_SCHED
+extern void android_vh_tick_nohz_idle_stop_tick_handler(void *unused, void *data);
+#endif
 /**
  * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
  *
@@ -1124,6 +1131,9 @@ void tick_nohz_idle_stop_tick(void)
 	ktime_t expires;
 
 	trace_android_vh_tick_nohz_idle_stop_tick(NULL);
+#ifdef CONFIG_HMBIRD_SCHED
+	android_vh_tick_nohz_idle_stop_tick_handler(NULL,NULL);
+#endif
 	/*
 	 * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
 	 * tick timer expiration time is known already.
diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore
deleted file mode 100644
index a3240f9f7eba..000000000000
--- a/tools/sched_ext/.gitignore
+++ /dev/null
@@ -1,9 +0,0 @@
-scx_example_simple
-scx_example_qmap
-scx_example_central
-scx_example_pair
-scx_example_flatcg
-scx_example_userland
-*.skel.h
-*.subskel.h
-/tools/
diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
deleted file mode 100644
index 73c43782837d..000000000000
--- a/tools/sched_ext/Makefile
+++ /dev/null
@@ -1,216 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-# Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
-include ../build/Build.include
-include ../scripts/Makefile.arch
-include ../scripts/Makefile.include
-
-ifneq ($(LLVM),)
-ifneq ($(filter %/,$(LLVM)),)
-LLVM_PREFIX := $(LLVM)
-else ifneq ($(filter -%,$(LLVM)),)
-LLVM_SUFFIX := $(LLVM)
-endif
-
-CLANG_TARGET_FLAGS_arm          := arm-linux-gnueabi
-CLANG_TARGET_FLAGS_arm64        := aarch64-linux-gnu
-CLANG_TARGET_FLAGS_hexagon      := hexagon-linux-musl
-CLANG_TARGET_FLAGS_m68k         := m68k-linux-gnu
-CLANG_TARGET_FLAGS_mips         := mipsel-linux-gnu
-CLANG_TARGET_FLAGS_powerpc      := powerpc64le-linux-gnu
-CLANG_TARGET_FLAGS_riscv        := riscv64-linux-gnu
-CLANG_TARGET_FLAGS_s390         := s390x-linux-gnu
-CLANG_TARGET_FLAGS_x86          := x86_64-linux-gnu
-CLANG_TARGET_FLAGS              := $(CLANG_TARGET_FLAGS_$(ARCH))
-
-ifeq ($(CROSS_COMPILE),)
-ifeq ($(CLANG_TARGET_FLAGS),)
-$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk
-else
-CLANG_FLAGS     += --target=$(CLANG_TARGET_FLAGS)
-endif # CLANG_TARGET_FLAGS
-else
-CLANG_FLAGS     += --target=$(notdir $(CROSS_COMPILE:%-=%))
-endif # CROSS_COMPILE
-
-CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as
-else
-CC := $(CROSS_COMPILE)gcc
-endif # LLVM
-
-CURDIR := $(abspath .)
-TOOLSDIR := $(abspath ..)
-LIBDIR := $(TOOLSDIR)/lib
-BPFDIR := $(LIBDIR)/bpf
-TOOLSINCDIR := $(TOOLSDIR)/include
-BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
-APIDIR := $(TOOLSINCDIR)/uapi
-GENDIR := $(abspath ../../include/generated)
-GENHDR := $(GENDIR)/autoconf.h
-
-SCRATCH_DIR := $(CURDIR)/tools
-BUILD_DIR := $(SCRATCH_DIR)/build
-INCLUDE_DIR := $(SCRATCH_DIR)/include
-BPFOBJ_DIR := $(BUILD_DIR)/libbpf
-BPFOBJ := $(BPFOBJ_DIR)/libbpf.a
-ifneq ($(CROSS_COMPILE),)
-HOST_BUILD_DIR		:= $(BUILD_DIR)/host
-HOST_SCRATCH_DIR	:= host-tools
-HOST_INCLUDE_DIR	:= $(HOST_SCRATCH_DIR)/include
-else
-HOST_BUILD_DIR		:= $(BUILD_DIR)
-HOST_SCRATCH_DIR	:= $(SCRATCH_DIR)
-HOST_INCLUDE_DIR	:= $(INCLUDE_DIR)
-endif
-HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a
-RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids
-DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool
-
-VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux)					\
-		     $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux)		\
-		     ../../vmlinux						\
-		     /sys/kernel/btf/vmlinux					\
-		     /boot/vmlinux-$(shell uname -r)
-VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
-ifeq ($(VMLINUX_BTF),)
-$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)")
-endif
-
-BPFTOOL ?= $(DEFAULT_BPFTOOL)
-
-ifneq ($(wildcard $(GENHDR)),)
-  GENFLAGS := -DHAVE_GENHDR
-endif
-
-CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS)			\
-	  -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR)				\
-	  -I$(TOOLSINCDIR) -I$(APIDIR)
-
-CARGOFLAGS := --release
-
-# Silence some warnings when compiled with clang
-ifneq ($(LLVM),)
-CFLAGS += -Wno-unused-command-line-argument
-endif
-
-LDFLAGS = -lelf -lz -lpthread
-
-IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null |				\
-			grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
-
-# Get Clang's default includes on this system, as opposed to those seen by
-# '-target bpf'. This fixes "missing" files on some architectures/distros,
-# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
-#
-# Use '-idirafter': Don't interfere with include mechanics except where the
-# build would have failed anyways.
-define get_sys_includes
-$(shell $(1) -v -E - </dev/null 2>&1 \
-	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
-$(shell $(1) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
-endef
-
-BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH)					\
-	     $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian)		\
-	     -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR)				\
-	     -I../../include							\
-	     $(call get_sys_includes,$(CLANG))					\
-	     -Wall -Wno-compare-distinct-pointer-types				\
-	     -O2 -mcpu=v3
-
-all: scx_example_simple scx_example_qmap scx_example_central scx_example_pair	\
-     scx_example_flatcg scx_example_userland atropos
-
-# sort removes libbpf duplicates when not cross-building
-MAKE_DIRS := $(sort $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf		\
-	       $(HOST_BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/resolve_btfids	\
-	       $(INCLUDE_DIR))
-
-$(MAKE_DIRS):
-	$(call msg,MKDIR,,$@)
-	$(Q)mkdir -p $@
-
-$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile)			\
-	   $(APIDIR)/linux/bpf.h						\
-	   | $(BUILD_DIR)/libbpf
-	$(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/	\
-		    EXTRA_CFLAGS='-g -O0 -fPIC'					\
-		    DESTDIR=$(SCRATCH_DIR) prefix= all install_headers
-
-$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile)	\
-		    $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool
-	$(Q)$(MAKE) $(submake_extras)  -C $(BPFTOOLDIR)				\
-		    ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD)		\
-		    EXTRA_CFLAGS='-g -O0'					\
-		    OUTPUT=$(HOST_BUILD_DIR)/bpftool/				\
-		    LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/			\
-		    LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/				\
-		    prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install-bin
-
-$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR)
-ifeq ($(VMLINUX_H),)
-	$(call msg,GEN,,$@)
-	$(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@
-else
-	$(call msg,CP,,$@)
-	$(Q)cp "$(VMLINUX_H)" $@
-endif
-
-%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h	\
-	| $(BPFOBJ)
-	$(call msg,CLNG-BPF,,$@)
-	$(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@
-
-%.skel.h: %.bpf.o $(BPFTOOL)
-	$(call msg,GEN-SKEL,,$@)
-	$(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $<
-	$(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o)
-	$(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o)
-	$(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o)
-	$(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $@
-	$(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(<:.bpf.o=) > $(@:.skel.h=.subskel.h)
-
-scx_example_simple: scx_example_simple.c scx_example_simple.skel.h user_exit_info.h
-	$(CC) $(CFLAGS) -c $< -o $@.o
-	$(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS)
-
-scx_example_qmap: scx_example_qmap.c scx_example_qmap.skel.h user_exit_info.h
-	$(CC) $(CFLAGS) -c $< -o $@.o
-	$(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS)
-
-scx_example_central: scx_example_central.c scx_example_central.skel.h user_exit_info.h
-	$(CC) $(CFLAGS) -c $< -o $@.o
-	$(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS)
-
-scx_example_pair: scx_example_pair.c scx_example_pair.skel.h user_exit_info.h
-	$(CC) $(CFLAGS) -c $< -o $@.o
-	$(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS)
-
-scx_example_flatcg: scx_example_flatcg.c scx_example_flatcg.skel.h user_exit_info.h
-	$(CC) $(CFLAGS) -c $< -o $@.o
-	$(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS)
-
-scx_example_userland: scx_example_userland.c scx_example_userland.skel.h	\
-		      scx_example_userland_common.h user_exit_info.h
-	$(CC) $(CFLAGS) -c $< -o $@.o
-	$(CC) -o $@ $@.o $(HOST_BPFOBJ) $(LDFLAGS)
-
-atropos: export RUSTFLAGS = -C link-args=-lzstd -C link-args=-lz -C link-args=-lelf -L $(BPFOBJ_DIR)
-atropos: export ATROPOS_CLANG = $(CLANG)
-atropos: export ATROPOS_BPF_CFLAGS = $(BPF_CFLAGS)
-atropos: $(INCLUDE_DIR)/vmlinux.h
-	cargo build --manifest-path=atropos/Cargo.toml $(CARGOFLAGS)
-
-clean:
-	cargo clean --manifest-path=atropos/Cargo.toml
-	rm -rf $(SCRATCH_DIR) $(HOST_SCRATCH_DIR)
-	rm -f *.o *.bpf.o *.skel.h *.subskel.h
-	rm -f scx_example_simple scx_example_qmap scx_example_central		\
-	      scx_example_pair scx_example_flatcg scx_example_userland
-
-.PHONY: all atropos clean
-
-# delete failed targets
-.DELETE_ON_ERROR:
-
-# keep intermediate (.skel.h, .bpf.o, etc) targets
-.SECONDARY:
diff --git a/tools/sched_ext/atropos/.gitignore b/tools/sched_ext/atropos/.gitignore
deleted file mode 100644
index 186dba259ec2..000000000000
--- a/tools/sched_ext/atropos/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-src/bpf/.output
-Cargo.lock
-target
diff --git a/tools/sched_ext/atropos/Cargo.toml b/tools/sched_ext/atropos/Cargo.toml
deleted file mode 100644
index 7462a836d53d..000000000000
--- a/tools/sched_ext/atropos/Cargo.toml
+++ /dev/null
@@ -1,28 +0,0 @@
-[package]
-name = "scx_atropos"
-version = "0.5.0"
-authors = ["Dan Schatzberg <dschatzberg@meta.com>", "Meta"]
-edition = "2021"
-description = "Userspace scheduling with BPF"
-license = "GPL-2.0-only"
-
-[dependencies]
-anyhow = "1.0.65"
-bitvec = { version = "1.0", features = ["serde"] }
-clap = { version = "4.1", features = ["derive", "env", "unicode", "wrap_help"] }
-ctrlc = { version = "3.1", features = ["termination"] }
-fb_procfs = { git = "https://github.com/facebookincubator/below.git", rev = "f305730"}
-hex = "0.4.3"
-libbpf-rs = "0.19.1"
-libbpf-sys = { version = "1.0.4", features = ["novendor", "static"] }
-libc = "0.2.137"
-log = "0.4.17"
-ordered-float = "3.4.0"
-simplelog = "0.12.0"
-
-[build-dependencies]
-bindgen = { version = "0.61.0", features = ["logging", "static"], default-features = false }
-libbpf-cargo = "0.13.0"
-
-[features]
-enable_backtrace = []
diff --git a/tools/sched_ext/atropos/build.rs b/tools/sched_ext/atropos/build.rs
deleted file mode 100644
index 26e792c5e17e..000000000000
--- a/tools/sched_ext/atropos/build.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-// This software may be used and distributed according to the terms of the
-// GNU General Public License version 2.
-extern crate bindgen;
-
-use std::env;
-use std::fs::create_dir_all;
-use std::path::Path;
-use std::path::PathBuf;
-
-use libbpf_cargo::SkeletonBuilder;
-
-const HEADER_PATH: &str = "src/bpf/atropos.h";
-
-fn bindgen_atropos() {
-    // Tell cargo to invalidate the built crate whenever the wrapper changes
-    println!("cargo:rerun-if-changed={}", HEADER_PATH);
-
-    // The bindgen::Builder is the main entry point
-    // to bindgen, and lets you build up options for
-    // the resulting bindings.
-    let bindings = bindgen::Builder::default()
-        // The input header we would like to generate
-        // bindings for.
-        .header(HEADER_PATH)
-        // Tell cargo to invalidate the built crate whenever any of the
-        // included header files changed.
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
-        // Finish the builder and generate the bindings.
-        .generate()
-        // Unwrap the Result and panic on failure.
-        .expect("Unable to generate bindings");
-
-    // Write the bindings to the $OUT_DIR/bindings.rs file.
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
-    bindings
-        .write_to_file(out_path.join("atropos-sys.rs"))
-        .expect("Couldn't write bindings!");
-}
-
-fn gen_bpf_sched(name: &str) {
-    let bpf_cflags = env::var("ATROPOS_BPF_CFLAGS").unwrap();
-    let clang = env::var("ATROPOS_CLANG").unwrap();
-    eprintln!("{}", clang);
-    let outpath = format!("./src/bpf/.output/{}.skel.rs", name);
-    let skel = Path::new(&outpath);
-    let src = format!("./src/bpf/{}.bpf.c", name);
-    SkeletonBuilder::new()
-        .source(src.clone())
-        .clang(clang)
-        .clang_args(bpf_cflags)
-        .build_and_generate(&skel)
-        .unwrap();
-    println!("cargo:rerun-if-changed={}", src);
-}
-
-fn main() {
-    bindgen_atropos();
-    // It's unfortunate we cannot use `OUT_DIR` to store the generated skeleton.
-    // Reasons are because the generated skeleton contains compiler attributes
-    // that cannot be `include!()`ed via macro. And we cannot use the `#[path = "..."]`
-    // trick either because you cannot yet `concat!(env!("OUT_DIR"), "/skel.rs")` inside
-    // the path attribute either (see https://github.com/rust-lang/rust/pull/83366).
-    //
-    // However, there is hope! When the above feature stabilizes we can clean this
-    // all up.
-    create_dir_all("./src/bpf/.output").unwrap();
-    gen_bpf_sched("atropos");
-}
diff --git a/tools/sched_ext/atropos/rustfmt.toml b/tools/sched_ext/atropos/rustfmt.toml
deleted file mode 100644
index b7258ed0a8d8..000000000000
--- a/tools/sched_ext/atropos/rustfmt.toml
+++ /dev/null
@@ -1,8 +0,0 @@
-# Get help on options with `rustfmt --help=config`
-# Please keep these in alphabetical order.
-edition = "2021"
-group_imports = "StdExternalCrate"
-imports_granularity = "Item"
-merge_derives = false
-use_field_init_shorthand = true
-version = "Two"
diff --git a/tools/sched_ext/atropos/src/atropos_sys.rs b/tools/sched_ext/atropos/src/atropos_sys.rs
deleted file mode 100644
index bbeaf856d40e..000000000000
--- a/tools/sched_ext/atropos/src/atropos_sys.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-// This software may be used and distributed according to the terms of the
-// GNU General Public License version 2.
-#![allow(non_upper_case_globals)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-#![allow(dead_code)]
-
-include!(concat!(env!("OUT_DIR"), "/atropos-sys.rs"));
diff --git a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c b/tools/sched_ext/atropos/src/bpf/atropos.bpf.c
deleted file mode 100644
index 3905a403e940..000000000000
--- a/tools/sched_ext/atropos/src/bpf/atropos.bpf.c
+++ /dev/null
@@ -1,743 +0,0 @@
-/* Copyright (c) Meta Platforms, Inc. and affiliates. */
-/*
- * This software may be used and distributed according to the terms of the
- * GNU General Public License version 2.
- *
- * Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF
- * part does simple round robin in each domain and the userspace part
- * calculates the load factor of each domain and tells the BPF part how to load
- * balance the domains.
- *
- * Every task has an entry in the task_data map which lists which domain the
- * task belongs to. When a task first enters the system (atropos_prep_enable),
- * they are round-robined to a domain.
- *
- * atropos_select_cpu is the primary scheduling logic, invoked when a task
- * becomes runnable. The lb_data map is populated by userspace to inform the BPF
- * scheduler that a task should be migrated to a new domain. Otherwise, the task
- * is scheduled in priority order as follows:
- * * The current core if the task was woken up synchronously and there are idle
- *   cpus in the system
- * * The previous core, if idle
- * * The pinned-to core if the task is pinned to a specific core
- * * Any idle cpu in the domain
- *
- * If none of the above conditions are met, then the task is enqueued to a
- * dispatch queue corresponding to the domain (atropos_enqueue).
- *
- * atropos_dispatch will attempt to consume a task from its domain's
- * corresponding dispatch queue (this occurs after scheduling any tasks directly
- * assigned to it due to the logic in atropos_select_cpu). If no task is found,
- * then greedy load stealing will attempt to find a task on another dispatch
- * queue to run.
- *
- * Load balancing is almost entirely handled by userspace. BPF populates the
- * task weight, dom mask and current dom in the task_data map and executes the
- * load balance based on userspace populating the lb_data map.
- */
-#include "../../../scx_common.bpf.h"
-#include "atropos.h"
-
-#include <errno.h>
-#include <stdbool.h>
-#include <string.h>
-#include <bpf/bpf_core_read.h>
-#include <bpf/bpf_helpers.h>
-#include <bpf/bpf_tracing.h>
-
-char _license[] SEC("license") = "GPL";
-
-/*
- * const volatiles are set during initialization and treated as consts by the
- * jit compiler.
- */
-
-/*
- * Domains and cpus
- */
-const volatile __u32 nr_doms = 32;	/* !0 for veristat, set during init */
-const volatile __u32 nr_cpus = 64;	/* !0 for veristat, set during init */
-const volatile __u32 cpu_dom_id_map[MAX_CPUS];
-const volatile __u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64];
-
-const volatile bool kthreads_local;
-const volatile bool fifo_sched;
-const volatile bool switch_partial;
-const volatile __u32 greedy_threshold;
-
-/* base slice duration */
-const volatile __u64 slice_us = 20000;
-
-/*
- * Exit info
- */
-int exit_type = SCX_EXIT_NONE;
-char exit_msg[SCX_EXIT_MSG_LEN];
-
-struct pcpu_ctx {
-	__u32 dom_rr_cur; /* used when scanning other doms */
-
-	/* libbpf-rs does not respect the alignment, so pad out the struct explicitly */
-	__u8 _padding[CACHELINE_SIZE - sizeof(u64)];
-} __attribute__((aligned(CACHELINE_SIZE)));
-
-struct pcpu_ctx pcpu_ctx[MAX_CPUS];
-
-/*
- * Domain context
- */
-struct dom_ctx {
-	struct bpf_cpumask __kptr *cpumask;
-	u64 vtime_now;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_ARRAY);
-	__type(key, u32);
-	__type(value, struct dom_ctx);
-	__uint(max_entries, MAX_DOMS);
-	__uint(map_flags, 0);
-} dom_ctx SEC(".maps");
-
-/*
- * Statistics
- */
-struct {
-	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
-	__uint(key_size, sizeof(u32));
-	__uint(value_size, sizeof(u64));
-	__uint(max_entries, ATROPOS_NR_STATS);
-} stats SEC(".maps");
-
-static inline void stat_add(enum stat_idx idx, u64 addend)
-{
-	u32 idx_v = idx;
-
-	u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v);
-	if (cnt_p)
-		(*cnt_p) += addend;
-}
-
-/* Map pid -> task_ctx */
-struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__type(key, pid_t);
-	__type(value, struct task_ctx);
-	__uint(max_entries, 1000000);
-	__uint(map_flags, 0);
-} task_data SEC(".maps");
-
-/*
- * This is populated from userspace to indicate which pids should be reassigned
- * to new doms.
- */
-struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__type(key, pid_t);
-	__type(value, u32);
-	__uint(max_entries, 1000);
-	__uint(map_flags, 0);
-} lb_data SEC(".maps");
-
-static inline bool vtime_before(u64 a, u64 b)
-{
-	return (s64)(a - b) < 0;
-}
-
-static bool task_set_dsq(struct task_ctx *task_ctx, struct task_struct *p,
-			 u32 new_dom_id)
-{
-	struct dom_ctx *old_domc, *new_domc;
-	struct bpf_cpumask *d_cpumask, *t_cpumask;
-	u32 old_dom_id = task_ctx->dom_id;
-	s64 vtime_delta;
-
-	old_domc = bpf_map_lookup_elem(&dom_ctx, &old_dom_id);
-	if (!old_domc) {
-		scx_bpf_error("No dom%u", old_dom_id);
-		return false;
-	}
-
-	vtime_delta = p->scx.dsq_vtime - old_domc->vtime_now;
-
-	new_domc = bpf_map_lookup_elem(&dom_ctx, &new_dom_id);
-	if (!new_domc) {
-		scx_bpf_error("No dom%u", new_dom_id);
-		return false;
-	}
-
-	d_cpumask = new_domc->cpumask;
-	if (!d_cpumask) {
-		scx_bpf_error("Failed to get domain %u cpumask kptr",
-			      new_dom_id);
-		return false;
-	}
-
-	t_cpumask = task_ctx->cpumask;
-	if (!t_cpumask) {
-		scx_bpf_error("Failed to look up task cpumask");
-		return false;
-	}
-
-	/*
-	 * set_cpumask might have happened between userspace requesting LB and
-	 * here and @p might not be able to run in @dom_id anymore. Verify.
-	 */
-	if (bpf_cpumask_intersects((const struct cpumask *)d_cpumask,
-				   p->cpus_ptr)) {
-		p->scx.dsq_vtime = new_domc->vtime_now + vtime_delta;
-		task_ctx->dom_id = new_dom_id;
-		bpf_cpumask_and(t_cpumask, (const struct cpumask *)d_cpumask,
-				p->cpus_ptr);
-	}
-
-	return task_ctx->dom_id == new_dom_id;
-}
-
-s32 BPF_STRUCT_OPS(atropos_select_cpu, struct task_struct *p, int prev_cpu,
-		   u32 wake_flags)
-{
-	s32 cpu;
-	pid_t pid = p->pid;
-	struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid);
-	struct bpf_cpumask *p_cpumask;
-
-	if (!task_ctx)
-		return -ENOENT;
-
-	if (kthreads_local &&
-	    (p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) {
-		cpu = prev_cpu;
-		stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1);
-		goto local;
-	}
-
-	/*
-	 * If WAKE_SYNC and the machine isn't fully saturated, wake up @p to the
-	 * local dsq of the waker.
-	 */
-	if (p->nr_cpus_allowed > 1 && (wake_flags & SCX_WAKE_SYNC)) {
-		struct task_struct *current = (void *)bpf_get_current_task();
-
-		if (!(BPF_CORE_READ(current, flags) & PF_EXITING) &&
-		    task_ctx->dom_id < MAX_DOMS) {
-			struct dom_ctx *domc;
-			struct bpf_cpumask *d_cpumask;
-			const struct cpumask *idle_cpumask;
-			bool has_idle;
-
-			domc = bpf_map_lookup_elem(&dom_ctx, &task_ctx->dom_id);
-			if (!domc) {
-				scx_bpf_error("Failed to find dom%u",
-					      task_ctx->dom_id);
-				return prev_cpu;
-			}
-			d_cpumask = domc->cpumask;
-			if (!d_cpumask) {
-				scx_bpf_error("Failed to acquire domain %u cpumask kptr",
-					      task_ctx->dom_id);
-				return prev_cpu;
-			}
-
-			idle_cpumask = scx_bpf_get_idle_cpumask();
-
-			has_idle = bpf_cpumask_intersects((const struct cpumask *)d_cpumask,
-							  idle_cpumask);
-
-			scx_bpf_put_idle_cpumask(idle_cpumask);
-
-			if (has_idle) {
-				cpu = bpf_get_smp_processor_id();
-				if (bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) {
-					stat_add(ATROPOS_STAT_WAKE_SYNC, 1);
-					goto local;
-				}
-			}
-		}
-	}
-
-	/* if the previous CPU is idle, dispatch directly to it */
-	if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) {
-		stat_add(ATROPOS_STAT_PREV_IDLE, 1);
-		cpu = prev_cpu;
-		goto local;
-	}
-
-	/* If only one core is allowed, dispatch */
-	if (p->nr_cpus_allowed == 1) {
-		stat_add(ATROPOS_STAT_PINNED, 1);
-		cpu = prev_cpu;
-		goto local;
-	}
-
-	p_cpumask = task_ctx->cpumask;
-	if (!p_cpumask)
-		return -ENOENT;
-
-	/* If there is an eligible idle CPU, dispatch directly */
-	cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask);
-	if (cpu >= 0) {
-		stat_add(ATROPOS_STAT_DIRECT_DISPATCH, 1);
-		goto local;
-	}
-
-	/*
-	 * @prev_cpu may be in a different domain. Returning an out-of-domain
-	 * CPU can lead to stalls as all in-domain CPUs may be idle by the time
-	 * @p gets enqueued.
-	 */
-	if (bpf_cpumask_test_cpu(prev_cpu, (const struct cpumask *)p_cpumask))
-		cpu = prev_cpu;
-	else
-		cpu = bpf_cpumask_any((const struct cpumask *)p_cpumask);
-
-	return cpu;
-
-local:
-	task_ctx->dispatch_local = true;
-	return cpu;
-}
-
-void BPF_STRUCT_OPS(atropos_enqueue, struct task_struct *p, u32 enq_flags)
-{
-	pid_t pid = p->pid;
-	struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid);
-	u32 *new_dom;
-
-	if (!task_ctx) {
-		scx_bpf_error("No task_ctx[%d]", pid);
-		return;
-	}
-
-	new_dom = bpf_map_lookup_elem(&lb_data, &pid);
-	if (new_dom && *new_dom != task_ctx->dom_id &&
-	    task_set_dsq(task_ctx, p, *new_dom)) {
-		struct bpf_cpumask *p_cpumask;
-		s32 cpu;
-
-		stat_add(ATROPOS_STAT_LOAD_BALANCE, 1);
-
-		/*
-		 * If dispatch_local is set, We own @p's idle state but we are
-		 * not gonna put the task in the associated local dsq which can
-		 * cause the CPU to stall. Kick it.
-		 */
-		if (task_ctx->dispatch_local) {
-			task_ctx->dispatch_local = false;
-			scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 0);
-		}
-
-		p_cpumask = task_ctx->cpumask;
-		if (!p_cpumask) {
-			scx_bpf_error("Failed to get task_ctx->cpumask");
-			return;
-		}
-		cpu = scx_bpf_pick_idle_cpu((const struct cpumask *)p_cpumask);
-
-		if (cpu >= 0)
-			scx_bpf_kick_cpu(cpu, 0);
-	}
-
-	if (task_ctx->dispatch_local) {
-		task_ctx->dispatch_local = false;
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_us * 1000, enq_flags);
-		return;
-	}
-
-	if (fifo_sched) {
-		scx_bpf_dispatch(p, task_ctx->dom_id, slice_us * 1000,
-				 enq_flags);
-	} else {
-		u64 vtime = p->scx.dsq_vtime;
-		u32 dom_id = task_ctx->dom_id;
-		struct dom_ctx *domc;
-
-		domc = bpf_map_lookup_elem(&dom_ctx, &dom_id);
-		if (!domc) {
-			scx_bpf_error("No dom[%u]", dom_id);
-			return;
-		}
-
-		/*
-		 * Limit the amount of budget that an idling task can accumulate
-		 * to one slice.
-		 */
-		if (vtime_before(vtime, domc->vtime_now - slice_us * 1000))
-			vtime = domc->vtime_now - slice_us * 1000;
-
-		scx_bpf_dispatch_vtime(p, task_ctx->dom_id, SCX_SLICE_DFL, vtime,
-				       enq_flags);
-	}
-}
-
-static u32 cpu_to_dom_id(s32 cpu)
-{
-	const volatile u32 *dom_idp;
-
-	if (nr_doms <= 1)
-		return 0;
-
-	dom_idp = MEMBER_VPTR(cpu_dom_id_map, [cpu]);
-	if (!dom_idp)
-		return MAX_DOMS;
-
-	return *dom_idp;
-}
-
-static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id)
-{
-	s32 cpu;
-
-	if (dom_id >= MAX_DOMS)
-		return false;
-
-	bpf_for(cpu, 0, nr_cpus) {
-		if (bpf_cpumask_test_cpu(cpu, cpumask) &&
-		    (dom_cpumasks[dom_id][cpu / 64] & (1LLU << (cpu % 64))))
-			return true;
-	}
-	return false;
-}
-
-static u32 dom_rr_next(s32 cpu)
-{
-	struct pcpu_ctx *pcpuc;
-	u32 dom_id;
-
-	pcpuc = MEMBER_VPTR(pcpu_ctx, [cpu]);
-	if (!pcpuc)
-		return 0;
-
-	dom_id = (pcpuc->dom_rr_cur + 1) % nr_doms;
-
-	if (dom_id == cpu_to_dom_id(cpu))
-		dom_id = (dom_id + 1) % nr_doms;
-
-	pcpuc->dom_rr_cur = dom_id;
-	return dom_id;
-}
-
-void BPF_STRUCT_OPS(atropos_dispatch, s32 cpu, struct task_struct *prev)
-{
-	u32 dom = cpu_to_dom_id(cpu);
-
-	if (scx_bpf_consume(dom)) {
-		stat_add(ATROPOS_STAT_DSQ_DISPATCH, 1);
-		return;
-	}
-
-	if (!greedy_threshold)
-		return;
-
-	bpf_repeat(nr_doms - 1) {
-		u32 dom_id = dom_rr_next(cpu);
-
-		if (scx_bpf_dsq_nr_queued(dom_id) >= greedy_threshold &&
-		    scx_bpf_consume(dom_id)) {
-			stat_add(ATROPOS_STAT_GREEDY, 1);
-			break;
-		}
-	}
-}
-
-void BPF_STRUCT_OPS(atropos_runnable, struct task_struct *p, u64 enq_flags)
-{
-	pid_t pid = p->pid;
-	struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid);
-
-	if (!task_ctx) {
-		scx_bpf_error("No task_ctx[%d]", pid);
-		return;
-	}
-
-	task_ctx->runnable_at = bpf_ktime_get_ns();
-}
-
-void BPF_STRUCT_OPS(atropos_running, struct task_struct *p)
-{
-	struct task_ctx *taskc;
-	struct dom_ctx *domc;
-	pid_t pid = p->pid;
-	u32 dom_id;
-
-	if (fifo_sched)
-		return;
-
-	taskc = bpf_map_lookup_elem(&task_data, &pid);
-	if (!taskc) {
-		scx_bpf_error("No task_ctx[%d]", pid);
-		return;
-	}
-	dom_id = taskc->dom_id;
-
-	domc = bpf_map_lookup_elem(&dom_ctx, &dom_id);
-	if (!domc) {
-		scx_bpf_error("No dom[%u]", dom_id);
-		return;
-	}
-
-	/*
-	 * Global vtime always progresses forward as tasks start executing. The
-	 * test and update can be performed concurrently from multiple CPUs and
-	 * thus racy. Any error should be contained and temporary. Let's just
-	 * live with it.
-	 */
-	if (vtime_before(domc->vtime_now, p->scx.dsq_vtime))
-		domc->vtime_now = p->scx.dsq_vtime;
-}
-
-void BPF_STRUCT_OPS(atropos_stopping, struct task_struct *p, bool runnable)
-{
-	if (fifo_sched)
-		return;
-
-	/* scale the execution time by the inverse of the weight and charge */
-	p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight;
-}
-
-void BPF_STRUCT_OPS(atropos_quiescent, struct task_struct *p, u64 deq_flags)
-{
-	pid_t pid = p->pid;
-	struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid);
-
-	if (!task_ctx) {
-		scx_bpf_error("No task_ctx[%d]", pid);
-		return;
-	}
-
-	task_ctx->runnable_for += bpf_ktime_get_ns() - task_ctx->runnable_at;
-	task_ctx->runnable_at = 0;
-}
-
-void BPF_STRUCT_OPS(atropos_set_weight, struct task_struct *p, u32 weight)
-{
-	pid_t pid = p->pid;
-	struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid);
-
-	if (!task_ctx) {
-		scx_bpf_error("No task_ctx[%d]", pid);
-		return;
-	}
-
-	task_ctx->weight = weight;
-}
-
-struct pick_task_domain_loop_ctx {
-	struct task_struct *p;
-	const struct cpumask *cpumask;
-	u64 dom_mask;
-	u32 dom_rr_base;
-	u32 dom_id;
-};
-
-static int pick_task_domain_loopfn(u32 idx, void *data)
-{
-	struct pick_task_domain_loop_ctx *lctx = data;
-	u32 dom_id = (lctx->dom_rr_base + idx) % nr_doms;
-
-	if (dom_id >= MAX_DOMS)
-		return 1;
-
-	if (cpumask_intersects_domain(lctx->cpumask, dom_id)) {
-		lctx->dom_mask |= 1LLU << dom_id;
-		if (lctx->dom_id == MAX_DOMS)
-			lctx->dom_id = dom_id;
-	}
-	return 0;
-}
-
-static u32 pick_task_domain(struct task_ctx *task_ctx, struct task_struct *p,
-			    const struct cpumask *cpumask)
-{
-	struct pick_task_domain_loop_ctx lctx = {
-		.p = p,
-		.cpumask = cpumask,
-		.dom_id = MAX_DOMS,
-	};
-	s32 cpu = bpf_get_smp_processor_id();
-
-	if (cpu < 0 || cpu >= MAX_CPUS)
-		return MAX_DOMS;
-
-	lctx.dom_rr_base = ++(pcpu_ctx[cpu].dom_rr_cur);
-
-	bpf_loop(nr_doms, pick_task_domain_loopfn, &lctx, 0);
-	task_ctx->dom_mask = lctx.dom_mask;
-
-	return lctx.dom_id;
-}
-
-static void task_set_domain(struct task_ctx *task_ctx, struct task_struct *p,
-			    const struct cpumask *cpumask)
-{
-	u32 dom_id = 0;
-
-	if (nr_doms > 1)
-		dom_id = pick_task_domain(task_ctx, p, cpumask);
-
-	if (!task_set_dsq(task_ctx, p, dom_id))
-		scx_bpf_error("Failed to set domain %d for %s[%d]",
-			      dom_id, p->comm, p->pid);
-}
-
-void BPF_STRUCT_OPS(atropos_set_cpumask, struct task_struct *p,
-		    const struct cpumask *cpumask)
-{
-	pid_t pid = p->pid;
-	struct task_ctx *task_ctx = bpf_map_lookup_elem(&task_data, &pid);
-	if (!task_ctx) {
-		scx_bpf_error("No task_ctx[%d]", pid);
-		return;
-	}
-
-	task_set_domain(task_ctx, p, cpumask);
-}
-
-s32 BPF_STRUCT_OPS(atropos_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
-{
-	struct bpf_cpumask *cpumask;
-	struct task_ctx task_ctx, *map_value;
-	long ret;
-	pid_t pid;
-
-	memset(&task_ctx, 0, sizeof(task_ctx));
-
-	pid = p->pid;
-	ret = bpf_map_update_elem(&task_data, &pid, &task_ctx, BPF_NOEXIST);
-	if (ret) {
-		stat_add(ATROPOS_STAT_TASK_GET_ERR, 1);
-		return ret;
-	}
-
-	/*
-	 * Read the entry from the map immediately so we can add the cpumask
-	 * with bpf_kptr_xchg().
-	 */
-	map_value = bpf_map_lookup_elem(&task_data, &pid);
-	if (!map_value)
-		/* Should never happen -- it was just inserted above. */
-		return -EINVAL;
-
-	cpumask = bpf_cpumask_create();
-	if (!cpumask) {
-		bpf_map_delete_elem(&task_data, &pid);
-		return -ENOMEM;
-	}
-
-	cpumask = bpf_kptr_xchg(&map_value->cpumask, cpumask);
-	if (cpumask) {
-		/* Should never happen as we just inserted it above. */
-		bpf_cpumask_release(cpumask);
-		bpf_map_delete_elem(&task_data, &pid);
-		return -EINVAL;
-	}
-
-	task_set_domain(map_value, p, p->cpus_ptr);
-
-	return 0;
-}
-
-void BPF_STRUCT_OPS(atropos_disable, struct task_struct *p)
-{
-	pid_t pid = p->pid;
-	long ret = bpf_map_delete_elem(&task_data, &pid);
-	if (ret) {
-		stat_add(ATROPOS_STAT_TASK_GET_ERR, 1);
-		return;
-	}
-}
-
-static int create_dom_dsq(u32 idx, void *data)
-{
-	struct dom_ctx domc_init = {}, *domc;
-	struct bpf_cpumask *cpumask;
-	u32 cpu, dom_id = idx;
-	s32 ret;
-
-	ret = scx_bpf_create_dsq(dom_id, -1);
-	if (ret < 0) {
-		scx_bpf_error("Failed to create dsq %u (%d)", dom_id, ret);
-		return 1;
-	}
-
-	ret = bpf_map_update_elem(&dom_ctx, &dom_id, &domc_init, 0);
-	if (ret) {
-		scx_bpf_error("Failed to add dom_ctx entry %u (%d)", dom_id, ret);
-		return 1;
-	}
-
-	domc = bpf_map_lookup_elem(&dom_ctx, &dom_id);
-	if (!domc) {
-		/* Should never happen, we just inserted it above. */
-		scx_bpf_error("No dom%u", dom_id);
-		return 1;
-	}
-
-	cpumask = bpf_cpumask_create();
-	if (!cpumask) {
-		scx_bpf_error("Failed to create BPF cpumask for domain %u", dom_id);
-		return 1;
-	}
-
-	for (cpu = 0; cpu < MAX_CPUS; cpu++) {
-		const volatile __u64 *dmask;
-
-		dmask = MEMBER_VPTR(dom_cpumasks, [dom_id][cpu / 64]);
-		if (!dmask) {
-			scx_bpf_error("array index error");
-			bpf_cpumask_release(cpumask);
-			return 1;
-		}
-
-		if (*dmask & (1LLU << (cpu % 64)))
-			bpf_cpumask_set_cpu(cpu, cpumask);
-	}
-
-	cpumask = bpf_kptr_xchg(&domc->cpumask, cpumask);
-	if (cpumask) {
-		scx_bpf_error("Domain %u was already present", dom_id);
-		bpf_cpumask_release(cpumask);
-		return 1;
-	}
-
-	return 0;
-}
-
-int BPF_STRUCT_OPS_SLEEPABLE(atropos_init)
-{
-	if (!switch_partial)
-		scx_bpf_switch_all();
-
-	bpf_loop(nr_doms, create_dom_dsq, NULL, 0);
-
-	for (u32 i = 0; i < nr_cpus; i++)
-		pcpu_ctx[i].dom_rr_cur = i;
-
-	return 0;
-}
-
-void BPF_STRUCT_OPS(atropos_exit, struct scx_exit_info *ei)
-{
-	bpf_probe_read_kernel_str(exit_msg, sizeof(exit_msg), ei->msg);
-	exit_type = ei->type;
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops atropos = {
-	.select_cpu = (void *)atropos_select_cpu,
-	.enqueue = (void *)atropos_enqueue,
-	.dispatch = (void *)atropos_dispatch,
-	.runnable = (void *)atropos_runnable,
-	.running = (void *)atropos_running,
-	.stopping = (void *)atropos_stopping,
-	.quiescent = (void *)atropos_quiescent,
-	.set_weight = (void *)atropos_set_weight,
-	.set_cpumask = (void *)atropos_set_cpumask,
-	.prep_enable = (void *)atropos_prep_enable,
-	.disable = (void *)atropos_disable,
-	.init = (void *)atropos_init,
-	.exit = (void *)atropos_exit,
-	.flags = 0,
-	.name = "atropos",
-};
diff --git a/tools/sched_ext/atropos/src/bpf/atropos.h b/tools/sched_ext/atropos/src/bpf/atropos.h
deleted file mode 100644
index addf29ca104a..000000000000
--- a/tools/sched_ext/atropos/src/bpf/atropos.h
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-// This software may be used and distributed according to the terms of the
-// GNU General Public License version 2.
-#ifndef __ATROPOS_H
-#define __ATROPOS_H
-
-#include <stdbool.h>
-#ifndef __kptr
-#ifdef __KERNEL__
-#error "__kptr_ref not defined in the kernel"
-#endif
-#define __kptr
-#endif
-
-#define	MAX_CPUS 512
-#define	MAX_DOMS 64 /* limited to avoid complex bitmask ops */
-#define	CACHELINE_SIZE 64
-
-/* Statistics */
-enum stat_idx {
-	ATROPOS_STAT_TASK_GET_ERR,
-	ATROPOS_STAT_WAKE_SYNC,
-	ATROPOS_STAT_PREV_IDLE,
-	ATROPOS_STAT_PINNED,
-	ATROPOS_STAT_DIRECT_DISPATCH,
-	ATROPOS_STAT_DSQ_DISPATCH,
-	ATROPOS_STAT_GREEDY,
-	ATROPOS_STAT_LOAD_BALANCE,
-	ATROPOS_STAT_LAST_TASK,
-	ATROPOS_NR_STATS,
-};
-
-struct task_ctx {
-	unsigned long long dom_mask; /* the domains this task can run on */
-	struct bpf_cpumask __kptr *cpumask;
-	unsigned int dom_id;
-	unsigned int weight;
-	unsigned long long runnable_at;
-	unsigned long long runnable_for;
-	bool dispatch_local;
-};
-
-#endif /* __ATROPOS_H */
diff --git a/tools/sched_ext/atropos/src/main.rs b/tools/sched_ext/atropos/src/main.rs
deleted file mode 100644
index 0d313662f713..000000000000
--- a/tools/sched_ext/atropos/src/main.rs
+++ /dev/null
@@ -1,942 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-// This software may be used and distributed according to the terms of the
-// GNU General Public License version 2.
-#[path = "bpf/.output/atropos.skel.rs"]
-mod atropos;
-pub use atropos::*;
-pub mod atropos_sys;
-
-use std::cell::Cell;
-use std::collections::{BTreeMap, BTreeSet};
-use std::ffi::CStr;
-use std::ops::Bound::{Included, Unbounded};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-use std::time::{Duration, SystemTime};
-
-use ::fb_procfs as procfs;
-use anyhow::{anyhow, bail, Context, Result};
-use bitvec::prelude::*;
-use clap::Parser;
-use log::{info, trace, warn};
-use ordered_float::OrderedFloat;
-
-/// Atropos is a multi-domain BPF / userspace hybrid scheduler where the BPF
-/// part does simple round robin in each domain and the userspace part
-/// calculates the load factor of each domain and tells the BPF part how to load
-/// balance the domains.
-
-/// This scheduler demonstrates dividing scheduling logic between BPF and
-/// userspace and using rust to build the userspace part. An earlier variant of
-/// this scheduler was used to balance across six domains, each representing a
-/// chiplet in a six-chiplet AMD processor, and could match the performance of
-/// production setup using CFS.
-#[derive(Debug, Parser)]
-struct Opts {
-    /// Scheduling slice duration in microseconds.
-    #[clap(short, long, default_value = "20000")]
-    slice_us: u64,
-
-    /// Monitoring and load balance interval in seconds.
-    #[clap(short, long, default_value = "2.0")]
-    interval: f64,
-
-    /// Build domains according to how CPUs are grouped at this cache level
-    /// as determined by /sys/devices/system/cpu/cpuX/cache/indexI/id.
-    #[clap(short = 'c', long, default_value = "3")]
-    cache_level: u32,
-
-    /// Instead of using cache locality, set the cpumask for each domain
-    /// manually, provide multiple --cpumasks, one for each domain. E.g.
-    /// --cpumasks 0xff_00ff --cpumasks 0xff00 will create two domains with
-    /// the corresponding CPUs belonging to each domain. Each CPU must
-    /// belong to precisely one domain.
-    #[clap(short = 'C', long, num_args = 1.., conflicts_with = "cache_level")]
-    cpumasks: Vec<String>,
-
-    /// When non-zero, enable greedy task stealing. When a domain is idle, a
-    /// cpu will attempt to steal tasks from a domain with at least
-    /// greedy_threshold tasks enqueued. These tasks aren't permanently
-    /// stolen from the domain.
-    #[clap(short, long, default_value = "4")]
-    greedy_threshold: u32,
-
-    /// The load decay factor. Every interval, the existing load is decayed
-    /// by this factor and new load is added. Must be in the range [0.0,
-    /// 0.99]. The smaller the value, the more sensitive load calculation
-    /// is to recent changes. When 0.0, history is ignored and the load
-    /// value from the latest period is used directly.
-    #[clap(short, long, default_value = "0.5")]
-    load_decay_factor: f64,
-
-    /// Disable load balancing. Unless disabled, periodically userspace will
-    /// calculate the load factor of each domain and instruct BPF which
-    /// processes to move.
-    #[clap(short, long, action = clap::ArgAction::SetTrue)]
-    no_load_balance: bool,
-
-    /// Put per-cpu kthreads directly into local dsq's.
-    #[clap(short, long, action = clap::ArgAction::SetTrue)]
-    kthreads_local: bool,
-
-    /// Use FIFO scheduling instead of weighted vtime scheduling.
-    #[clap(short, long, action = clap::ArgAction::SetTrue)]
-    fifo_sched: bool,
-
-    /// If specified, only tasks which have their scheduling policy set to
-    /// SCHED_EXT using sched_setscheduler(2) are switched. Otherwise, all
-    /// tasks are switched.
-    #[clap(short, long, action = clap::ArgAction::SetTrue)]
-    partial: bool,
-
-    /// Enable verbose output including libbpf details. Specify multiple
-    /// times to increase verbosity.
-    #[clap(short, long, action = clap::ArgAction::Count)]
-    verbose: u8,
-}
-
-fn read_total_cpu(reader: &mut procfs::ProcReader) -> Result<procfs::CpuStat> {
-    Ok(reader
-        .read_stat()
-        .context("Failed to read procfs")?
-        .total_cpu
-        .ok_or_else(|| anyhow!("Could not read total cpu stat in proc"))?)
-}
-
-fn now_monotonic() -> u64 {
-    let mut time = libc::timespec {
-        tv_sec: 0,
-        tv_nsec: 0,
-    };
-    let ret = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut time) };
-    assert!(ret == 0);
-    time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64
-}
-
-fn clear_map(map: &mut libbpf_rs::Map) {
-    // XXX: libbpf_rs has some design flaw that make it impossible to
-    // delete while iterating despite it being safe so we alias it here
-    let deleter: &mut libbpf_rs::Map = unsafe { &mut *(map as *mut _) };
-    for key in map.keys() {
-        let _ = deleter.delete(&key);
-    }
-}
-
-#[derive(Debug)]
-struct TaskLoad {
-    runnable_for: u64,
-    load: f64,
-}
-
-#[derive(Debug)]
-struct TaskInfo {
-    pid: i32,
-    dom_mask: u64,
-    migrated: Cell<bool>,
-}
-
-struct LoadBalancer<'a, 'b, 'c> {
-    maps: AtroposMapsMut<'a>,
-    task_loads: &'b mut BTreeMap<i32, TaskLoad>,
-    nr_doms: usize,
-    load_decay_factor: f64,
-
-    tasks_by_load: Vec<BTreeMap<OrderedFloat<f64>, TaskInfo>>,
-    load_avg: f64,
-    dom_loads: Vec<f64>,
-
-    imbal: Vec<f64>,
-    doms_to_push: BTreeMap<OrderedFloat<f64>, u32>,
-    doms_to_pull: BTreeMap<OrderedFloat<f64>, u32>,
-
-    nr_lb_data_errors: &'c mut u64,
-}
-
-impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> {
-    const LOAD_IMBAL_HIGH_RATIO: f64 = 0.10;
-    const LOAD_IMBAL_REDUCTION_MIN_RATIO: f64 = 0.1;
-    const LOAD_IMBAL_PUSH_MAX_RATIO: f64 = 0.50;
-
-    fn new(
-        maps: AtroposMapsMut<'a>,
-        task_loads: &'b mut BTreeMap<i32, TaskLoad>,
-        nr_doms: usize,
-        load_decay_factor: f64,
-        nr_lb_data_errors: &'c mut u64,
-    ) -> Self {
-        Self {
-            maps,
-            task_loads,
-            nr_doms,
-            load_decay_factor,
-
-            tasks_by_load: (0..nr_doms).map(|_| BTreeMap::<_, _>::new()).collect(),
-            load_avg: 0f64,
-            dom_loads: vec![0.0; nr_doms],
-
-            imbal: vec![0.0; nr_doms],
-            doms_to_pull: BTreeMap::new(),
-            doms_to_push: BTreeMap::new(),
-
-            nr_lb_data_errors,
-        }
-    }
-
-    fn read_task_loads(&mut self, period: Duration) -> Result<()> {
-        let now_mono = now_monotonic();
-        let task_data = self.maps.task_data();
-        let mut this_task_loads = BTreeMap::<i32, TaskLoad>::new();
-        let mut load_sum = 0.0f64;
-        self.dom_loads = vec![0f64; self.nr_doms];
-
-        for key in task_data.keys() {
-            if let Some(task_ctx_vec) = task_data
-                .lookup(&key, libbpf_rs::MapFlags::ANY)
-                .context("Failed to lookup task_data")?
-            {
-                let task_ctx =
-                    unsafe { &*(task_ctx_vec.as_slice().as_ptr() as *const atropos_sys::task_ctx) };
-                let pid = i32::from_ne_bytes(
-                    key.as_slice()
-                        .try_into()
-                        .context("Invalid key length in task_data map")?,
-                );
-
-                let (this_at, this_for, weight) = unsafe {
-                    (
-                        std::ptr::read_volatile(&task_ctx.runnable_at as *const u64),
-                        std::ptr::read_volatile(&task_ctx.runnable_for as *const u64),
-                        std::ptr::read_volatile(&task_ctx.weight as *const u32),
-                    )
-                };
-
-                let (mut delta, prev_load) = match self.task_loads.get(&pid) {
-                    Some(prev) => (this_for - prev.runnable_for, Some(prev.load)),
-                    None => (this_for, None),
-                };
-
-                // Non-zero this_at indicates that the task is currently
-                // runnable. Note that we read runnable_at and runnable_for
-                // without any synchronization and there is a small window
-                // where we end up misaccounting. While this can cause
-                // temporary error, it's unlikely to cause any noticeable
-                // misbehavior especially given the load value clamping.
-                if this_at > 0 && this_at < now_mono {
-                    delta += now_mono - this_at;
-                }
-
-                delta = delta.min(period.as_nanos() as u64);
-                let this_load = (weight as f64 * delta as f64 / period.as_nanos() as f64)
-                    .clamp(0.0, weight as f64);
-
-                let this_load = match prev_load {
-                    Some(prev_load) => {
-                        prev_load * self.load_decay_factor
-                            + this_load * (1.0 - self.load_decay_factor)
-                    }
-                    None => this_load,
-                };
-
-                this_task_loads.insert(
-                    pid,
-                    TaskLoad {
-                        runnable_for: this_for,
-                        load: this_load,
-                    },
-                );
-
-                load_sum += this_load;
-                self.dom_loads[task_ctx.dom_id as usize] += this_load;
-                // Only record pids that are eligible for load balancing
-                if task_ctx.dom_mask == (1u64 << task_ctx.dom_id) {
-                    continue;
-                }
-                self.tasks_by_load[task_ctx.dom_id as usize].insert(
-                    OrderedFloat(this_load),
-                    TaskInfo {
-                        pid,
-                        dom_mask: task_ctx.dom_mask,
-                        migrated: Cell::new(false),
-                    },
-                );
-            }
-        }
-
-        self.load_avg = load_sum / self.nr_doms as f64;
-        *self.task_loads = this_task_loads;
-        Ok(())
-    }
-
-    // To balance dom loads we identify doms with lower and higher load than average
-    fn calculate_dom_load_balance(&mut self) -> Result<()> {
-        for (dom, dom_load) in self.dom_loads.iter().enumerate() {
-            let imbal = dom_load - self.load_avg;
-            if imbal.abs() >= self.load_avg * Self::LOAD_IMBAL_HIGH_RATIO {
-                if imbal > 0f64 {
-                    self.doms_to_push.insert(OrderedFloat(imbal), dom as u32);
-                } else {
-                    self.doms_to_pull.insert(OrderedFloat(-imbal), dom as u32);
-                }
-                self.imbal[dom] = imbal;
-            }
-        }
-        Ok(())
-    }
-
-    // Find the first candidate pid which hasn't already been migrated and
-    // can run in @pull_dom.
-    fn find_first_candidate<'d, I>(tasks_by_load: I, pull_dom: u32) -> Option<(f64, &'d TaskInfo)>
-    where
-        I: IntoIterator<Item = (&'d OrderedFloat<f64>, &'d TaskInfo)>,
-    {
-        match tasks_by_load
-            .into_iter()
-            .skip_while(|(_, task)| task.migrated.get() || task.dom_mask & (1 << pull_dom) == 0)
-            .next()
-        {
-            Some((OrderedFloat(load), task)) => Some((*load, task)),
-            None => None,
-        }
-    }
-
-    fn pick_victim(
-        &self,
-        (push_dom, to_push): (u32, f64),
-        (pull_dom, to_pull): (u32, f64),
-    ) -> Option<(&TaskInfo, f64)> {
-        let to_xfer = to_pull.min(to_push);
-
-        trace!(
-            "considering dom {}@{:.2} -> {}@{:.2}",
-            push_dom,
-            to_push,
-            pull_dom,
-            to_pull
-        );
-
-        let calc_new_imbal = |xfer: f64| (to_push - xfer).abs() + (to_pull - xfer).abs();
-
-        trace!(
-            "to_xfer={:.2} tasks_by_load={:?}",
-            to_xfer,
-            &self.tasks_by_load[push_dom as usize]
-        );
-
-        // We want to pick a task to transfer from push_dom to pull_dom to
-        // maximize the reduction of load imbalance between the two. IOW,
-        // pick a task which has the closest load value to $to_xfer that can
-        // be migrated. Find such task by locating the first migratable task
-        // while scanning left from $to_xfer and the counterpart while
-        // scanning right and picking the better of the two.
-        let (load, task, new_imbal) = match (
-            Self::find_first_candidate(
-                self.tasks_by_load[push_dom as usize]
-                    .range((Unbounded, Included(&OrderedFloat(to_xfer))))
-                    .rev(),
-                pull_dom,
-            ),
-            Self::find_first_candidate(
-                self.tasks_by_load[push_dom as usize]
-                    .range((Included(&OrderedFloat(to_xfer)), Unbounded)),
-                pull_dom,
-            ),
-        ) {
-            (None, None) => return None,
-            (Some((load, task)), None) | (None, Some((load, task))) => {
-                (load, task, calc_new_imbal(load))
-            }
-            (Some((load0, task0)), Some((load1, task1))) => {
-                let (new_imbal0, new_imbal1) = (calc_new_imbal(load0), calc_new_imbal(load1));
-                if new_imbal0 <= new_imbal1 {
-                    (load0, task0, new_imbal0)
-                } else {
-                    (load1, task1, new_imbal1)
-                }
-            }
-        };
-
-        // If the best candidate can't reduce the imbalance, there's nothing
-        // to do for this pair.
-        let old_imbal = to_push + to_pull;
-        if old_imbal * (1.0 - Self::LOAD_IMBAL_REDUCTION_MIN_RATIO) < new_imbal {
-            trace!(
-                "skipping pid {}, dom {} -> {} won't improve imbal {:.2} -> {:.2}",
-                task.pid,
-                push_dom,
-                pull_dom,
-                old_imbal,
-                new_imbal
-            );
-            return None;
-        }
-
-        trace!(
-            "migrating pid {}, dom {} -> {}, imbal={:.2} -> {:.2}",
-            task.pid,
-            push_dom,
-            pull_dom,
-            old_imbal,
-            new_imbal,
-        );
-
-        Some((task, load))
-    }
-
-    // Actually execute the load balancing. Concretely this writes pid -> dom
-    // entries into the lb_data map for bpf side to consume.
-    fn load_balance(&mut self) -> Result<()> {
-        clear_map(self.maps.lb_data());
-
-        trace!("imbal={:?}", &self.imbal);
-        trace!("doms_to_push={:?}", &self.doms_to_push);
-        trace!("doms_to_pull={:?}", &self.doms_to_pull);
-
-        // Push from the most imbalanced to least.
-        while let Some((OrderedFloat(mut to_push), push_dom)) = self.doms_to_push.pop_last() {
-            let push_max = self.dom_loads[push_dom as usize] * Self::LOAD_IMBAL_PUSH_MAX_RATIO;
-            let mut pushed = 0f64;
-
-            // Transfer tasks from push_dom to reduce imbalance.
-            loop {
-                let last_pushed = pushed;
-
-                // Pull from the most imbalaned to least.
-                let mut doms_to_pull = BTreeMap::<_, _>::new();
-                std::mem::swap(&mut self.doms_to_pull, &mut doms_to_pull);
-                let mut pull_doms = doms_to_pull.into_iter().rev().collect::<Vec<(_, _)>>();
-
-                for (to_pull, pull_dom) in pull_doms.iter_mut() {
-                    if let Some((task, load)) =
-                        self.pick_victim((push_dom, to_push), (*pull_dom, f64::from(*to_pull)))
-                    {
-                        // Execute migration.
-                        task.migrated.set(true);
-                        to_push -= load;
-                        *to_pull -= load;
-                        pushed += load;
-
-                        // Ask BPF code to execute the migration.
-                        let pid = task.pid;
-                        let cpid = (pid as libc::pid_t).to_ne_bytes();
-                        if let Err(e) = self.maps.lb_data().update(
-                            &cpid,
-                            &pull_dom.to_ne_bytes(),
-                            libbpf_rs::MapFlags::NO_EXIST,
-                        ) {
-                            warn!(
-                                "Failed to update lb_data map for pid={} error={:?}",
-                                pid, &e
-                            );
-                            *self.nr_lb_data_errors += 1;
-                        }
-
-                        // Always break after a successful migration so that
-                        // the pulling domains are always considered in the
-                        // descending imbalance order.
-                        break;
-                    }
-                }
-
-                pull_doms
-                    .into_iter()
-                    .map(|(k, v)| self.doms_to_pull.insert(k, v))
-                    .count();
-
-                // Stop repeating if nothing got transferred or pushed enough.
-                if pushed == last_pushed || pushed >= push_max {
-                    break;
-                }
-            }
-        }
-        Ok(())
-    }
-}
-
-struct Scheduler<'a> {
-    skel: AtroposSkel<'a>,
-    struct_ops: Option<libbpf_rs::Link>,
-
-    nr_cpus: usize,
-    nr_doms: usize,
-    load_decay_factor: f64,
-    balance_load: bool,
-
-    proc_reader: procfs::ProcReader,
-
-    prev_at: SystemTime,
-    prev_total_cpu: procfs::CpuStat,
-    task_loads: BTreeMap<i32, TaskLoad>,
-
-    nr_lb_data_errors: u64,
-}
-
-impl<'a> Scheduler<'a> {
-    // Returns Vec of cpuset for each dq and a vec of dq for each cpu
-    fn parse_cpusets(
-        cpumasks: &[String],
-        nr_cpus: usize,
-    ) -> Result<(Vec<BitVec<u64, Lsb0>>, Vec<i32>)> {
-        if cpumasks.len() > atropos_sys::MAX_DOMS as usize {
-            bail!(
-                "Number of requested DSQs ({}) is greater than MAX_DOMS ({})",
-                cpumasks.len(),
-                atropos_sys::MAX_DOMS
-            );
-        }
-        let mut cpus = vec![-1i32; nr_cpus];
-        let mut cpusets =
-            vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; cpumasks.len()];
-        for (dq, cpumask) in cpumasks.iter().enumerate() {
-            let hex_str = {
-                let mut tmp_str = cpumask
-                    .strip_prefix("0x")
-                    .unwrap_or(cpumask)
-                    .replace('_', "");
-                if tmp_str.len() % 2 != 0 {
-                    tmp_str = "0".to_string() + &tmp_str;
-                }
-                tmp_str
-            };
-            let byte_vec = hex::decode(&hex_str)
-                .with_context(|| format!("Failed to parse cpumask: {}", cpumask))?;
-
-            for (index, &val) in byte_vec.iter().rev().enumerate() {
-                let mut v = val;
-                while v != 0 {
-                    let lsb = v.trailing_zeros() as usize;
-                    v &= !(1 << lsb);
-                    let cpu = index * 8 + lsb;
-                    if cpu > nr_cpus {
-                        bail!(
-                            concat!(
-                                "Found cpu ({}) in cpumask ({}) which is larger",
-                                " than the number of cpus on the machine ({})"
-                            ),
-                            cpu,
-                            cpumask,
-                            nr_cpus
-                        );
-                    }
-                    if cpus[cpu] != -1 {
-                        bail!(
-                            "Found cpu ({}) with dq ({}) but also in cpumask ({})",
-                            cpu,
-                            cpus[cpu],
-                            cpumask
-                        );
-                    }
-                    cpus[cpu] = dq as i32;
-                    cpusets[dq].set(cpu, true);
-                }
-            }
-            cpusets[dq].set_uninitialized(false);
-        }
-
-        for (cpu, &dq) in cpus.iter().enumerate() {
-            if dq < 0 {
-                bail!(
-                "Cpu {} not assigned to any dq. Make sure it is covered by some --cpumasks argument.",
-                cpu
-            );
-            }
-        }
-
-        Ok((cpusets, cpus))
-    }
-
-    // Returns Vec of cpuset for each dq and a vec of dq for each cpu
-    fn cpusets_from_cache(
-        level: u32,
-        nr_cpus: usize,
-    ) -> Result<(Vec<BitVec<u64, Lsb0>>, Vec<i32>)> {
-        let mut cpu_to_cache = vec![]; // (cpu_id, cache_id)
-        let mut cache_ids = BTreeSet::<u32>::new();
-        let mut nr_not_found = 0;
-
-        // Build cpu -> cache ID mapping.
-        for cpu in 0..nr_cpus {
-            let path = format!("/sys/devices/system/cpu/cpu{}/cache/index{}/id", cpu, level);
-            let id = match std::fs::read_to_string(&path) {
-                Ok(val) => val
-                    .trim()
-                    .parse::<u32>()
-                    .with_context(|| format!("Failed to parse {:?}'s content {:?}", &path, &val))?,
-                Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
-                    nr_not_found += 1;
-                    0
-                }
-                Err(e) => return Err(e).with_context(|| format!("Failed to open {:?}", &path)),
-            };
-
-            cpu_to_cache.push(id);
-            cache_ids.insert(id);
-        }
-
-        if nr_not_found > 1 {
-            warn!(
-                "Couldn't determine level {} cache IDs for {} CPUs out of {}, assigned to cache ID 0",
-                level, nr_not_found, nr_cpus
-            );
-        }
-
-        // Cache IDs may have holes. Assign consecutive domain IDs to
-        // existing cache IDs.
-        let mut cache_to_dom = BTreeMap::<u32, u32>::new();
-        let mut nr_doms = 0;
-        for cache_id in cache_ids.iter() {
-            cache_to_dom.insert(*cache_id, nr_doms);
-            nr_doms += 1;
-        }
-
-        if nr_doms > atropos_sys::MAX_DOMS {
-            bail!(
-                "Total number of doms {} is greater than MAX_DOMS ({})",
-                nr_doms,
-                atropos_sys::MAX_DOMS
-            );
-        }
-
-        // Build and return dom -> cpumask and cpu -> dom mappings.
-        let mut cpusets =
-            vec![bitvec![u64, Lsb0; 0; atropos_sys::MAX_CPUS as usize]; nr_doms as usize];
-        let mut cpu_to_dom = vec![];
-
-        for cpu in 0..nr_cpus {
-            let dom_id = cache_to_dom[&cpu_to_cache[cpu]];
-            cpusets[dom_id as usize].set(cpu, true);
-            cpu_to_dom.push(dom_id as i32);
-        }
-
-        Ok((cpusets, cpu_to_dom))
-    }
-
-    fn init(opts: &Opts) -> Result<Self> {
-        // Open the BPF prog first for verification.
-        let mut skel_builder = AtroposSkelBuilder::default();
-        skel_builder.obj_builder.debug(opts.verbose > 0);
-        let mut skel = skel_builder.open().context("Failed to open BPF program")?;
-
-        let nr_cpus = libbpf_rs::num_possible_cpus().unwrap();
-        if nr_cpus > atropos_sys::MAX_CPUS as usize {
-            bail!(
-                "nr_cpus ({}) is greater than MAX_CPUS ({})",
-                nr_cpus,
-                atropos_sys::MAX_CPUS
-            );
-        }
-
-        // Initialize skel according to @opts.
-        let (cpusets, cpus) = if opts.cpumasks.len() > 0 {
-            Self::parse_cpusets(&opts.cpumasks, nr_cpus)?
-        } else {
-            Self::cpusets_from_cache(opts.cache_level, nr_cpus)?
-        };
-        let nr_doms = cpusets.len();
-        skel.rodata().nr_doms = nr_doms as u32;
-        skel.rodata().nr_cpus = nr_cpus as u32;
-
-        for (cpu, dom) in cpus.iter().enumerate() {
-            skel.rodata().cpu_dom_id_map[cpu] = *dom as u32;
-        }
-
-        for (dom, cpuset) in cpusets.iter().enumerate() {
-            let raw_cpuset_slice = cpuset.as_raw_slice();
-            let dom_cpumask_slice = &mut skel.rodata().dom_cpumasks[dom];
-            let (left, _) = dom_cpumask_slice.split_at_mut(raw_cpuset_slice.len());
-            left.clone_from_slice(cpuset.as_raw_slice());
-            let cpumask_str = dom_cpumask_slice
-                .iter()
-                .take((nr_cpus + 63) / 64)
-                .rev()
-                .fold(String::new(), |acc, x| format!("{} {:016X}", acc, x));
-            info!(
-                "DOM[{:02}] cpumask{} ({} cpus)",
-                dom,
-                &cpumask_str,
-                cpuset.count_ones()
-            );
-        }
-
-        skel.rodata().slice_us = opts.slice_us;
-        skel.rodata().kthreads_local = opts.kthreads_local;
-        skel.rodata().fifo_sched = opts.fifo_sched;
-        skel.rodata().switch_partial = opts.partial;
-        skel.rodata().greedy_threshold = opts.greedy_threshold;
-
-        // Attach.
-        let mut skel = skel.load().context("Failed to load BPF program")?;
-        skel.attach().context("Failed to attach BPF program")?;
-        let struct_ops = Some(
-            skel.maps_mut()
-                .atropos()
-                .attach_struct_ops()
-                .context("Failed to attach atropos struct ops")?,
-        );
-        info!("Atropos Scheduler Attached");
-
-        // Other stuff.
-        let mut proc_reader = procfs::ProcReader::new();
-        let prev_total_cpu = read_total_cpu(&mut proc_reader)?;
-
-        Ok(Self {
-            skel,
-            struct_ops, // should be held to keep it attached
-
-            nr_cpus,
-            nr_doms,
-            load_decay_factor: opts.load_decay_factor.clamp(0.0, 0.99),
-            balance_load: !opts.no_load_balance,
-
-            proc_reader,
-
-            prev_at: SystemTime::now(),
-            prev_total_cpu,
-            task_loads: BTreeMap::new(),
-
-            nr_lb_data_errors: 0,
-        })
-    }
-
-    fn get_cpu_busy(&mut self) -> Result<f64> {
-        let total_cpu = read_total_cpu(&mut self.proc_reader)?;
-        let busy = match (&self.prev_total_cpu, &total_cpu) {
-            (
-                procfs::CpuStat {
-                    user_usec: Some(prev_user),
-                    nice_usec: Some(prev_nice),
-                    system_usec: Some(prev_system),
-                    idle_usec: Some(prev_idle),
-                    iowait_usec: Some(prev_iowait),
-                    irq_usec: Some(prev_irq),
-                    softirq_usec: Some(prev_softirq),
-                    stolen_usec: Some(prev_stolen),
-                    guest_usec: _,
-                    guest_nice_usec: _,
-                },
-                procfs::CpuStat {
-                    user_usec: Some(curr_user),
-                    nice_usec: Some(curr_nice),
-                    system_usec: Some(curr_system),
-                    idle_usec: Some(curr_idle),
-                    iowait_usec: Some(curr_iowait),
-                    irq_usec: Some(curr_irq),
-                    softirq_usec: Some(curr_softirq),
-                    stolen_usec: Some(curr_stolen),
-                    guest_usec: _,
-                    guest_nice_usec: _,
-                },
-            ) => {
-                let idle_usec = curr_idle - prev_idle;
-                let iowait_usec = curr_iowait - prev_iowait;
-                let user_usec = curr_user - prev_user;
-                let system_usec = curr_system - prev_system;
-                let nice_usec = curr_nice - prev_nice;
-                let irq_usec = curr_irq - prev_irq;
-                let softirq_usec = curr_softirq - prev_softirq;
-                let stolen_usec = curr_stolen - prev_stolen;
-
-                let busy_usec =
-                    user_usec + system_usec + nice_usec + irq_usec + softirq_usec + stolen_usec;
-                let total_usec = idle_usec + busy_usec + iowait_usec;
-                busy_usec as f64 / total_usec as f64
-            }
-            _ => {
-                bail!("Some procfs stats are not populated!");
-            }
-        };
-
-        self.prev_total_cpu = total_cpu;
-        Ok(busy)
-    }
-
-    fn read_bpf_stats(&mut self) -> Result<Vec<u64>> {
-        let mut maps = self.skel.maps_mut();
-        let stats_map = maps.stats();
-        let mut stats: Vec<u64> = Vec::new();
-        let zero_vec = vec![vec![0u8; stats_map.value_size() as usize]; self.nr_cpus];
-
-        for stat in 0..atropos_sys::stat_idx_ATROPOS_NR_STATS {
-            let cpu_stat_vec = stats_map
-                .lookup_percpu(&(stat as u32).to_ne_bytes(), libbpf_rs::MapFlags::ANY)
-                .with_context(|| format!("Failed to lookup stat {}", stat))?
-                .expect("per-cpu stat should exist");
-            let sum = cpu_stat_vec
-                .iter()
-                .map(|val| {
-                    u64::from_ne_bytes(
-                        val.as_slice()
-                            .try_into()
-                            .expect("Invalid value length in stat map"),
-                    )
-                })
-                .sum();
-            stats_map
-                .update_percpu(
-                    &(stat as u32).to_ne_bytes(),
-                    &zero_vec,
-                    libbpf_rs::MapFlags::ANY,
-                )
-                .context("Failed to zero stat")?;
-            stats.push(sum);
-        }
-        Ok(stats)
-    }
-
-    fn report(
-        &self,
-        stats: &Vec<u64>,
-        cpu_busy: f64,
-        processing_dur: Duration,
-        load_avg: f64,
-        dom_loads: &Vec<f64>,
-        imbal: &Vec<f64>,
-    ) {
-        let stat = |idx| stats[idx as usize];
-        let total = stat(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC)
-            + stat(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE)
-            + stat(atropos_sys::stat_idx_ATROPOS_STAT_PINNED)
-            + stat(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH)
-            + stat(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH)
-            + stat(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY)
-            + stat(atropos_sys::stat_idx_ATROPOS_STAT_LAST_TASK);
-
-        info!(
-            "cpu={:6.1} load_avg={:7.1} bal={} task_err={} lb_data_err={} proc={:?}ms",
-            cpu_busy * 100.0,
-            load_avg,
-            stats[atropos_sys::stat_idx_ATROPOS_STAT_LOAD_BALANCE as usize],
-            stats[atropos_sys::stat_idx_ATROPOS_STAT_TASK_GET_ERR as usize],
-            self.nr_lb_data_errors,
-            processing_dur.as_millis(),
-        );
-
-        let stat_pct = |idx| stat(idx) as f64 / total as f64 * 100.0;
-
-        info!(
-            "tot={:6} wsync={:4.1} prev_idle={:4.1} pin={:4.1} dir={:4.1} dq={:4.1} greedy={:4.1}",
-            total,
-            stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_WAKE_SYNC),
-            stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PREV_IDLE),
-            stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_PINNED),
-            stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DIRECT_DISPATCH),
-            stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_DSQ_DISPATCH),
-            stat_pct(atropos_sys::stat_idx_ATROPOS_STAT_GREEDY),
-        );
-
-        for i in 0..self.nr_doms {
-            info!(
-                "DOM[{:02}] load={:7.1} to_pull={:7.1} to_push={:7.1}",
-                i,
-                dom_loads[i],
-                if imbal[i] < 0.0 { -imbal[i] } else { 0.0 },
-                if imbal[i] > 0.0 { imbal[i] } else { 0.0 },
-            );
-        }
-    }
-
-    fn step(&mut self) -> Result<()> {
-        let started_at = std::time::SystemTime::now();
-        let bpf_stats = self.read_bpf_stats()?;
-        let cpu_busy = self.get_cpu_busy()?;
-
-        let mut lb = LoadBalancer::new(
-            self.skel.maps_mut(),
-            &mut self.task_loads,
-            self.nr_doms,
-            self.load_decay_factor,
-            &mut self.nr_lb_data_errors,
-        );
-
-        lb.read_task_loads(started_at.duration_since(self.prev_at)?)?;
-        lb.calculate_dom_load_balance()?;
-
-        if self.balance_load {
-            lb.load_balance()?;
-        }
-
-        // Extract fields needed for reporting and drop lb to release
-        // mutable borrows.
-        let (load_avg, dom_loads, imbal) = (lb.load_avg, lb.dom_loads, lb.imbal);
-
-        self.report(
-            &bpf_stats,
-            cpu_busy,
-            std::time::SystemTime::now().duration_since(started_at)?,
-            load_avg,
-            &dom_loads,
-            &imbal,
-        );
-
-        self.prev_at = started_at;
-        Ok(())
-    }
-
-    fn read_bpf_exit_type(&mut self) -> i32 {
-        unsafe { std::ptr::read_volatile(&self.skel.bss().exit_type as *const _) }
-    }
-
-    fn report_bpf_exit_type(&mut self) -> Result<()> {
-        // Report msg if EXT_OPS_EXIT_ERROR.
-        match self.read_bpf_exit_type() {
-            0 => Ok(()),
-            etype if etype == 2 => {
-                let cstr = unsafe { CStr::from_ptr(self.skel.bss().exit_msg.as_ptr() as *const _) };
-                let msg = cstr
-                    .to_str()
-                    .context("Failed to convert exit msg to string")
-                    .unwrap();
-                bail!("BPF exit_type={} msg={}", etype, msg);
-            }
-            etype => {
-                info!("BPF exit_type={}", etype);
-                Ok(())
-            }
-        }
-    }
-}
-
-impl<'a> Drop for Scheduler<'a> {
-    fn drop(&mut self) {
-        if let Some(struct_ops) = self.struct_ops.take() {
-            drop(struct_ops);
-        }
-    }
-}
-
-fn main() -> Result<()> {
-    let opts = Opts::parse();
-
-    let llv = match opts.verbose {
-        0 => simplelog::LevelFilter::Info,
-        1 => simplelog::LevelFilter::Debug,
-        _ => simplelog::LevelFilter::Trace,
-    };
-    let mut lcfg = simplelog::ConfigBuilder::new();
-    lcfg.set_time_level(simplelog::LevelFilter::Error)
-        .set_location_level(simplelog::LevelFilter::Off)
-        .set_target_level(simplelog::LevelFilter::Off)
-        .set_thread_level(simplelog::LevelFilter::Off);
-    simplelog::TermLogger::init(
-        llv,
-        lcfg.build(),
-        simplelog::TerminalMode::Stderr,
-        simplelog::ColorChoice::Auto,
-    )?;
-
-    let shutdown = Arc::new(AtomicBool::new(false));
-    let shutdown_clone = shutdown.clone();
-    ctrlc::set_handler(move || {
-        shutdown_clone.store(true, Ordering::Relaxed);
-    })
-    .context("Error setting Ctrl-C handler")?;
-
-    let mut sched = Scheduler::init(&opts)?;
-
-    while !shutdown.load(Ordering::Relaxed) && sched.read_bpf_exit_type() == 0 {
-        std::thread::sleep(Duration::from_secs_f64(opts.interval));
-        sched.step()?;
-    }
-
-    sched.report_bpf_exit_type()
-}
diff --git a/tools/sched_ext/gnu/stubs.h b/tools/sched_ext/gnu/stubs.h
deleted file mode 100644
index 719225b16626..000000000000
--- a/tools/sched_ext/gnu/stubs.h
+++ /dev/null
@@ -1 +0,0 @@
-/* dummy .h to trick /usr/include/features.h to work with 'clang -target bpf' */
diff --git a/tools/sched_ext/scx_common.bpf.h b/tools/sched_ext/scx_common.bpf.h
deleted file mode 100644
index e56de9dc86f2..000000000000
--- a/tools/sched_ext/scx_common.bpf.h
+++ /dev/null
@@ -1,288 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#ifndef __SCHED_EXT_COMMON_BPF_H
-#define __SCHED_EXT_COMMON_BPF_H
-
-#include "vmlinux.h"
-#include <bpf/bpf_helpers.h>
-#include <bpf/bpf_tracing.h>
-#include <linux/errno.h>
-#include "user_exit_info.h"
-
-#define PF_KTHREAD			0x00200000	/* I am a kernel thread */
-#define PF_EXITING			0x00000004
-#define CLOCK_MONOTONIC			1
-
-/*
- * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can
- * lead to really confusing misbehaviors. Let's trigger a build failure.
- */
-static inline void ___vmlinux_h_sanity_check___(void)
-{
-	_Static_assert(SCX_DSQ_FLAG_BUILTIN,
-		       "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole");
-}
-
-void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym;
-
-static inline __attribute__((format(printf, 1, 2)))
-void ___scx_bpf_error_format_checker(const char *fmt, ...) {}
-
-/*
- * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
- * instead of an array of u64. Note that __param[] must have at least one
- * element to keep the verifier happy.
- */
-#define scx_bpf_error(fmt, args...)						\
-({										\
-	static char ___fmt[] = fmt;						\
-	unsigned long long ___param[___bpf_narg(args) ?: 1] = {};		\
-										\
-	_Pragma("GCC diagnostic push")						\
-	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"")			\
-	___bpf_fill(___param, args);						\
-	_Pragma("GCC diagnostic pop")						\
-										\
-	scx_bpf_error_bstr(___fmt, ___param, sizeof(___param));			\
-										\
-	___scx_bpf_error_format_checker(fmt, ##args);				\
-})
-
-void scx_bpf_switch_all(void) __ksym;
-s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
-bool scx_bpf_consume(u64 dsq_id) __ksym;
-u32 scx_bpf_dispatch_nr_slots(void) __ksym;
-void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym;
-void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym;
-void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
-s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym;
-bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym;
-s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed) __ksym;
-const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym;
-const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym;
-void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym;
-void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
-bool scx_bpf_task_running(const struct task_struct *p) __ksym;
-s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
-struct cgroup *scx_bpf_task_cgroup(struct task_struct *p) __ksym;
-u32 scx_bpf_reenqueue_local(void) __ksym;
-
-#define BPF_STRUCT_OPS(name, args...)						\
-SEC("struct_ops/"#name)								\
-BPF_PROG(name, ##args)
-
-#define BPF_STRUCT_OPS_SLEEPABLE(name, args...)					\
-SEC("struct_ops.s/"#name)							\
-BPF_PROG(name, ##args)
-
-/**
- * MEMBER_VPTR - Obtain the verified pointer to a struct or array member
- * @base: struct or array to index
- * @member: dereferenced member (e.g. ->field, [idx0][idx1], ...)
- *
- * The verifier often gets confused by the instruction sequence the compiler
- * generates for indexing struct fields or arrays. This macro forces the
- * compiler to generate a code sequence which first calculates the byte offset,
- * checks it against the struct or array size and add that byte offset to
- * generate the pointer to the member to help the verifier.
- *
- * Ideally, we want to abort if the calculated offset is out-of-bounds. However,
- * BPF currently doesn't support abort, so evaluate to NULL instead. The caller
- * must check for NULL and take appropriate action to appease the verifier. To
- * avoid confusing the verifier, it's best to check for NULL and dereference
- * immediately.
- *
- *	vptr = MEMBER_VPTR(my_array, [i][j]);
- *	if (!vptr)
- *		return error;
- *	*vptr = new_value;
- */
-#define MEMBER_VPTR(base, member) (typeof(base member) *)({			\
-	u64 __base = (u64)base;							\
-	u64 __addr = (u64)&(base member) - __base;				\
-	asm volatile (								\
-		"if %0 <= %[max] goto +2\n"					\
-		"%0 = 0\n"							\
-		"goto +1\n"							\
-		"%0 += %1\n"							\
-		: "+r"(__addr)							\
-		: "r"(__base),							\
-		  [max]"i"(sizeof(base) - sizeof(base member)));		\
-	__addr;									\
-})
-
-/*
- * BPF core and other generic helpers
- */
-
-/* list and rbtree */
-#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
-#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
-
-void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
-void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
-
-#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
-#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
-
-void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym;
-void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym;
-struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
-struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
-struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
-				      struct bpf_rb_node *node) __ksym;
-void bpf_rbtree_add(struct bpf_rb_root *root, struct bpf_rb_node *node,
-		    bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) __ksym;
-struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
-
-/* task */
-struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
-struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
-void bpf_task_release(struct task_struct *p) __ksym;
-
-/* cgroup */
-struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym;
-void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
-struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym;
-
-/* cpumask */
-struct bpf_cpumask *bpf_cpumask_create(void) __ksym;
-struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym;
-void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym;
-u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym;
-u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym;
-void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
-void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
-bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym;
-bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
-bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
-void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym;
-void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym;
-bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1,
-		     const struct cpumask *src2) __ksym;
-void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1,
-		    const struct cpumask *src2) __ksym;
-void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1,
-		     const struct cpumask *src2) __ksym;
-bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym;
-bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym;
-bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym;
-bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym;
-bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym;
-void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym;
-u32 bpf_cpumask_any(const struct cpumask *cpumask) __ksym;
-u32 bpf_cpumask_any_and(const struct cpumask *src1, const struct cpumask *src2) __ksym;
-
-/* rcu */
-void bpf_rcu_read_lock(void) __ksym;
-void bpf_rcu_read_unlock(void) __ksym;
-
-/* BPF core iterators from tools/testing/selftests/bpf/progs/bpf_misc.h */
-struct bpf_iter_num;
-
-extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __ksym;
-extern int *bpf_iter_num_next(struct bpf_iter_num *it) __ksym;
-extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __ksym;
-
-#ifndef bpf_for_each
-/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for
- * using BPF open-coded iterators without having to write mundane explicit
- * low-level loop logic. Instead, it provides for()-like generic construct
- * that can be used pretty naturally. E.g., for some hypothetical cgroup
- * iterator, you'd write:
- *
- * struct cgroup *cg, *parent_cg = <...>;
- *
- * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) {
- *     bpf_printk("Child cgroup id = %d", cg->cgroup_id);
- *     if (cg->cgroup_id == 123)
- *         break;
- * }
- *
- * I.e., it looks almost like high-level for each loop in other languages,
- * supports continue/break, and is verifiable by BPF verifier.
- *
- * For iterating integers, the difference betwen bpf_for_each(num, i, N, M)
- * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to
- * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int
- * *`, not just `int`. So for integers bpf_for() is more convenient.
- *
- * Note: this macro relies on C99 feature of allowing to declare variables
- * inside for() loop, bound to for() loop lifetime. It also utilizes GCC
- * extension: __attribute__((cleanup(<func>))), supported by both GCC and
- * Clang.
- */
-#define bpf_for_each(type, cur, args...) for (							\
-	/* initialize and define destructor */							\
-	struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */,	\
-						    cleanup(bpf_iter_##type##_destroy))),	\
-	/* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */		\
-			       *___p __attribute__((unused)) = (				\
-					bpf_iter_##type##_new(&___it, ##args),			\
-	/* this is a workaround for Clang bug: it currently doesn't emit BTF */			\
-	/* for bpf_iter_##type##_destroy() when used from cleanup() attribute */		\
-					(void)bpf_iter_##type##_destroy, (void *)0);		\
-	/* iteration and termination check */							\
-	(((cur) = bpf_iter_##type##_next(&___it)));						\
-)
-#endif /* bpf_for_each */
-
-#ifndef bpf_for
-/* bpf_for(i, start, end) implements a for()-like looping construct that sets
- * provided integer variable *i* to values starting from *start* through,
- * but not including, *end*. It also proves to BPF verifier that *i* belongs
- * to range [start, end), so this can be used for accessing arrays without
- * extra checks.
- *
- * Note: *start* and *end* are assumed to be expressions with no side effects
- * and whose values do not change throughout bpf_for() loop execution. They do
- * not have to be statically known or constant, though.
- *
- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
- * loop bound variables and cleanup attribute, supported by GCC and Clang.
- */
-#define bpf_for(i, start, end) for (								\
-	/* initialize and define destructor */							\
-	struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */	\
-						 cleanup(bpf_iter_num_destroy))),		\
-	/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */		\
-			    *___p __attribute__((unused)) = (					\
-				bpf_iter_num_new(&___it, (start), (end)),			\
-	/* this is a workaround for Clang bug: it currently doesn't emit BTF */			\
-	/* for bpf_iter_num_destroy() when used from cleanup() attribute */			\
-				(void)bpf_iter_num_destroy, (void *)0);				\
-	({											\
-		/* iteration step */								\
-		int *___t = bpf_iter_num_next(&___it);						\
-		/* termination and bounds check */						\
-		(___t && ((i) = *___t, (i) >= (start) && (i) < (end)));				\
-	});											\
-)
-#endif /* bpf_for */
-
-#ifndef bpf_repeat
-/* bpf_repeat(N) performs N iterations without exposing iteration number
- *
- * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for()
- * loop bound variables and cleanup attribute, supported by GCC and Clang.
- */
-#define bpf_repeat(N) for (									\
-	/* initialize and define destructor */							\
-	struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */	\
-						 cleanup(bpf_iter_num_destroy))),		\
-	/* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */		\
-			    *___p __attribute__((unused)) = (					\
-				bpf_iter_num_new(&___it, 0, (N)),				\
-	/* this is a workaround for Clang bug: it currently doesn't emit BTF */			\
-	/* for bpf_iter_num_destroy() when used from cleanup() attribute */			\
-				(void)bpf_iter_num_destroy, (void *)0);				\
-	bpf_iter_num_next(&___it);								\
-	/* nothing here  */									\
-)
-#endif /* bpf_repeat */
-
-#endif	/* __SCHED_EXT_COMMON_BPF_H */
diff --git a/tools/sched_ext/scx_example_central.bpf.c b/tools/sched_ext/scx_example_central.bpf.c
deleted file mode 100644
index 4cec04b4c2ed..000000000000
--- a/tools/sched_ext/scx_example_central.bpf.c
+++ /dev/null
@@ -1,334 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A central FIFO sched_ext scheduler which demonstrates the followings:
- *
- * a. Making all scheduling decisions from one CPU:
- *
- *    The central CPU is the only one making scheduling decisions. All other
- *    CPUs kick the central CPU when they run out of tasks to run.
- *
- *    There is one global BPF queue and the central CPU schedules all CPUs by
- *    dispatching from the global queue to each CPU's local dsq from dispatch().
- *    This isn't the most straightforward. e.g. It'd be easier to bounce
- *    through per-CPU BPF queues. The current design is chosen to maximally
- *    utilize and verify various SCX mechanisms such as LOCAL_ON dispatching.
- *
- * b. Tickless operation
- *
- *    All tasks are dispatched with the infinite slice which allows stopping the
- *    ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full
- *    parameter. The tickless operation can be observed through
- *    /proc/interrupts.
- *
- *    Periodic switching is enforced by a periodic timer checking all CPUs and
- *    preempting them as necessary. Unfortunately, BPF timer currently doesn't
- *    have a way to pin to a specific CPU, so the periodic timer isn't pinned to
- *    the central CPU.
- *
- * c. Preemption
- *
- *    Kthreads are unconditionally queued to the head of a matching local dsq
- *    and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always
- *    prioritized over user threads, which is required for ensuring forward
- *    progress as e.g. the periodic timer may run on a ksoftirqd and if the
- *    ksoftirqd gets starved by a user thread, there may not be anything else to
- *    vacate that user thread.
- *
- *    SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the
- *    next tasks.
- *
- * This scheduler is designed to maximize usage of various SCX mechanisms. A
- * more practical implementation would likely put the scheduling loop outside
- * the central CPU's dispatch() path and add some form of priority mechanism.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#include "scx_common.bpf.h"
-
-char _license[] SEC("license") = "GPL";
-
-enum {
-	FALLBACK_DSQ_ID		= 0,
-	MAX_CPUS		= 4096,
-	MS_TO_NS		= 1000LLU * 1000,
-	TIMER_INTERVAL_NS	= 1 * MS_TO_NS,
-};
-
-const volatile bool switch_partial;
-const volatile s32 central_cpu;
-const volatile u32 nr_cpu_ids = 64;	/* !0 for veristat, set during init */
-
-u64 nr_total, nr_locals, nr_queued, nr_lost_pids;
-u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries;
-u64 nr_overflows;
-
-struct user_exit_info uei;
-
-struct {
-	__uint(type, BPF_MAP_TYPE_QUEUE);
-	__uint(max_entries, 4096);
-	__type(value, s32);
-} central_q SEC(".maps");
-
-/* can't use percpu map due to bad lookups */
-static bool cpu_gimme_task[MAX_CPUS];
-static u64 cpu_started_at[MAX_CPUS];
-
-struct central_timer {
-	struct bpf_timer timer;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_ARRAY);
-	__uint(max_entries, 1);
-	__type(key, u32);
-	__type(value, struct central_timer);
-} central_timer SEC(".maps");
-
-static bool vtime_before(u64 a, u64 b)
-{
-	return (s64)(a - b) < 0;
-}
-
-s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p,
-		   s32 prev_cpu, u64 wake_flags)
-{
-	/*
-	 * Steer wakeups to the central CPU as much as possible to avoid
-	 * disturbing other CPUs. It's safe to blindly return the central cpu as
-	 * select_cpu() is a hint and if @p can't be on it, the kernel will
-	 * automatically pick a fallback CPU.
-	 */
-	return central_cpu;
-}
-
-void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags)
-{
-	s32 pid = p->pid;
-
-	__sync_fetch_and_add(&nr_total, 1);
-
-	/*
-	 * Push per-cpu kthreads at the head of local dsq's and preempt the
-	 * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked
-	 * behind other threads which is necessary for forward progress
-	 * guarantee as we depend on the BPF timer which may run from ksoftirqd.
-	 */
-	if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) {
-		__sync_fetch_and_add(&nr_locals, 1);
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF,
-				 enq_flags | SCX_ENQ_PREEMPT);
-		return;
-	}
-
-	if (bpf_map_push_elem(&central_q, &pid, 0)) {
-		__sync_fetch_and_add(&nr_overflows, 1);
-		scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags);
-		return;
-	}
-
-	__sync_fetch_and_add(&nr_queued, 1);
-
-	if (!scx_bpf_task_running(p))
-		scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
-}
-
-static bool dispatch_to_cpu(s32 cpu)
-{
-	struct task_struct *p;
-	s32 pid;
-
-	bpf_repeat(BPF_MAX_LOOPS) {
-		if (bpf_map_pop_elem(&central_q, &pid))
-			break;
-
-		__sync_fetch_and_sub(&nr_queued, 1);
-
-		p = bpf_task_from_pid(pid);
-		if (!p) {
-			__sync_fetch_and_add(&nr_lost_pids, 1);
-			continue;
-		}
-
-		/*
-		 * If we can't run the task at the top, do the dumb thing and
-		 * bounce it to the fallback dsq.
-		 */
-		if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) {
-			__sync_fetch_and_add(&nr_mismatches, 1);
-			scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0);
-			bpf_task_release(p);
-			continue;
-		}
-
-		/* dispatch to local and mark that @cpu doesn't need more */
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0);
-
-		if (cpu != central_cpu)
-			scx_bpf_kick_cpu(cpu, 0);
-
-		bpf_task_release(p);
-		return true;
-	}
-
-	return false;
-}
-
-void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev)
-{
-	if (cpu == central_cpu) {
-		/* dispatch for all other CPUs first */
-		__sync_fetch_and_add(&nr_dispatches, 1);
-
-		bpf_for(cpu, 0, nr_cpu_ids) {
-			bool *gimme;
-
-			if (!scx_bpf_dispatch_nr_slots())
-				break;
-
-			/* central's gimme is never set */
-			gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]);
-			if (gimme && !*gimme)
-				continue;
-
-			if (dispatch_to_cpu(cpu))
-				*gimme = false;
-		}
-
-		/*
-		 * Retry if we ran out of dispatch buffer slots as we might have
-		 * skipped some CPUs and also need to dispatch for self. The ext
-		 * core automatically retries if the local dsq is empty but we
-		 * can't rely on that as we're dispatching for other CPUs too.
-		 * Kick self explicitly to retry.
-		 */
-		if (!scx_bpf_dispatch_nr_slots()) {
-			__sync_fetch_and_add(&nr_retries, 1);
-			scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
-			return;
-		}
-
-		/* look for a task to run on the central CPU */
-		if (scx_bpf_consume(FALLBACK_DSQ_ID))
-			return;
-		dispatch_to_cpu(central_cpu);
-	} else {
-		bool *gimme;
-
-		if (scx_bpf_consume(FALLBACK_DSQ_ID))
-			return;
-
-		gimme = MEMBER_VPTR(cpu_gimme_task, [cpu]);
-		if (gimme)
-			*gimme = true;
-
-		/*
-		 * Force dispatch on the scheduling CPU so that it finds a task
-		 * to run for us.
-		 */
-		scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
-	}
-}
-
-void BPF_STRUCT_OPS(central_running, struct task_struct *p)
-{
-	s32 cpu = scx_bpf_task_cpu(p);
-	u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]);
-	if (started_at)
-		*started_at = bpf_ktime_get_ns() ?: 1;	/* 0 indicates idle */
-}
-
-void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable)
-{
-	s32 cpu = scx_bpf_task_cpu(p);
-	u64 *started_at = MEMBER_VPTR(cpu_started_at, [cpu]);
-	if (started_at)
-		*started_at = 0;
-}
-
-static int central_timerfn(void *map, int *key, struct bpf_timer *timer)
-{
-	u64 now = bpf_ktime_get_ns();
-	u64 nr_to_kick = nr_queued;
-	s32 i;
-
-	bpf_for(i, 0, nr_cpu_ids) {
-		s32 cpu = (nr_timers + i) % nr_cpu_ids;
-		u64 *started_at;
-
-		if (cpu == central_cpu)
-			continue;
-
-		/* kick iff the current one exhausted its slice */
-		started_at = MEMBER_VPTR(cpu_started_at, [cpu]);
-		if (started_at && *started_at &&
-		    vtime_before(now, *started_at + SCX_SLICE_DFL))
-			continue;
-
-		/* and there's something pending */
-		if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) ||
-		    scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu))
-			;
-		else if (nr_to_kick)
-			nr_to_kick--;
-		else
-			continue;
-
-		scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT);
-	}
-
-	scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
-
-	bpf_timer_start(timer, TIMER_INTERVAL_NS, 0);
-	__sync_fetch_and_add(&nr_timers, 1);
-	return 0;
-}
-
-int BPF_STRUCT_OPS_SLEEPABLE(central_init)
-{
-	u32 key = 0;
-	struct bpf_timer *timer;
-	int ret;
-
-	if (!switch_partial)
-		scx_bpf_switch_all();
-
-	ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1);
-	if (ret)
-		return ret;
-
-	timer = bpf_map_lookup_elem(&central_timer, &key);
-	if (!timer)
-		return -ESRCH;
-
-	bpf_timer_init(timer, &central_timer, CLOCK_MONOTONIC);
-	bpf_timer_set_callback(timer, central_timerfn);
-	ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0);
-	return ret;
-}
-
-void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei)
-{
-	uei_record(&uei, ei);
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops central_ops = {
-	/*
-	 * We are offloading all scheduling decisions to the central CPU and
-	 * thus being the last task on a given CPU doesn't mean anything
-	 * special. Enqueue the last tasks like any other tasks.
-	 */
-	.flags			= SCX_OPS_ENQ_LAST,
-
-	.select_cpu		= (void *)central_select_cpu,
-	.enqueue		= (void *)central_enqueue,
-	.dispatch		= (void *)central_dispatch,
-	.running		= (void *)central_running,
-	.stopping		= (void *)central_stopping,
-	.init			= (void *)central_init,
-	.exit			= (void *)central_exit,
-	.name			= "central",
-};
diff --git a/tools/sched_ext/scx_example_central.c b/tools/sched_ext/scx_example_central.c
deleted file mode 100644
index 7ad591cbdc65..000000000000
--- a/tools/sched_ext/scx_example_central.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <unistd.h>
-#include <signal.h>
-#include <assert.h>
-#include <libgen.h>
-#include <bpf/bpf.h>
-#include "user_exit_info.h"
-#include "scx_example_central.skel.h"
-
-const char help_fmt[] =
-"A central FIFO sched_ext scheduler.\n"
-"\n"
-"See the top-level comment in .bpf.c for more details.\n"
-"\n"
-"Usage: %s [-c CPU] [-p]\n"
-"\n"
-"  -c CPU        Override the central CPU (default: 0)\n"
-"  -p            Switch only tasks on SCHED_EXT policy intead of all\n"
-"  -h            Display this help and exit\n";
-
-static volatile int exit_req;
-
-static void sigint_handler(int dummy)
-{
-	exit_req = 1;
-}
-
-int main(int argc, char **argv)
-{
-	struct scx_example_central *skel;
-	struct bpf_link *link;
-	u64 seq = 0;
-	s32 opt;
-
-	signal(SIGINT, sigint_handler);
-	signal(SIGTERM, sigint_handler);
-
-	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
-
-	skel = scx_example_central__open();
-	assert(skel);
-
-	skel->rodata->central_cpu = 0;
-	skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus();
-
-	while ((opt = getopt(argc, argv, "c:ph")) != -1) {
-		switch (opt) {
-		case 'c':
-			skel->rodata->central_cpu = strtoul(optarg, NULL, 0);
-			break;
-		case 'p':
-			skel->rodata->switch_partial = true;
-			break;
-		default:
-			fprintf(stderr, help_fmt, basename(argv[0]));
-			return opt != 'h';
-		}
-	}
-
-	assert(!scx_example_central__load(skel));
-
-	link = bpf_map__attach_struct_ops(skel->maps.central_ops);
-	assert(link);
-
-	while (!exit_req && !uei_exited(&skel->bss->uei)) {
-		printf("[SEQ %lu]\n", seq++);
-		printf("total   :%10lu    local:%10lu   queued:%10lu  lost:%10lu\n",
-		       skel->bss->nr_total,
-		       skel->bss->nr_locals,
-		       skel->bss->nr_queued,
-		       skel->bss->nr_lost_pids);
-		printf("timer   :%10lu dispatch:%10lu mismatch:%10lu retry:%10lu\n",
-		       skel->bss->nr_timers,
-		       skel->bss->nr_dispatches,
-		       skel->bss->nr_mismatches,
-		       skel->bss->nr_retries);
-		printf("overflow:%10lu\n",
-		       skel->bss->nr_overflows);
-		fflush(stdout);
-		sleep(1);
-	}
-
-	bpf_link__destroy(link);
-	uei_print(&skel->bss->uei);
-	scx_example_central__destroy(skel);
-	return 0;
-}
diff --git a/tools/sched_ext/scx_example_flatcg.bpf.c b/tools/sched_ext/scx_example_flatcg.bpf.c
deleted file mode 100644
index f6078b9a681f..000000000000
--- a/tools/sched_ext/scx_example_flatcg.bpf.c
+++ /dev/null
@@ -1,872 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A demo sched_ext flattened cgroup hierarchy scheduler. It implements
- * hierarchical weight-based cgroup CPU control by flattening the cgroup
- * hierarchy into a single layer by compounding the active weight share at each
- * level. Consider the following hierarchy with weights in parentheses:
- *
- * R + A (100) + B (100)
- *   |         \ C (100)
- *   \ D (200)
- *
- * Ignoring the root and threaded cgroups, only B, C and D can contain tasks.
- * Let's say all three have runnable tasks. The total share that each of these
- * three cgroups is entitled to can be calculated by compounding its share at
- * each level.
- *
- * For example, B is competing against C and in that competition its share is
- * 100/(100+100) == 1/2. At its parent level, A is competing against D and A's
- * share in that competition is 200/(200+100) == 1/3. B's eventual share in the
- * system can be calculated by multiplying the two shares, 1/2 * 1/3 == 1/6. C's
- * eventual shaer is the same at 1/6. D is only competing at the top level and
- * its share is 200/(100+200) == 2/3.
- *
- * So, instead of hierarchically scheduling level-by-level, we can consider it
- * as B, C and D competing each other with respective share of 1/6, 1/6 and 2/3
- * and keep updating the eventual shares as the cgroups' runnable states change.
- *
- * This flattening of hierarchy can bring a substantial performance gain when
- * the cgroup hierarchy is nested multiple levels. in a simple benchmark using
- * wrk[8] on apache serving a CGI script calculating sha1sum of a small file, it
- * outperforms CFS by ~3% with CPU controller disabled and by ~10% with two
- * apache instances competing with 2:1 weight ratio nested four level deep.
- *
- * However, the gain comes at the cost of not being able to properly handle
- * thundering herd of cgroups. For example, if many cgroups which are nested
- * behind a low priority parent cgroup wake up around the same time, they may be
- * able to consume more CPU cycles than they are entitled to. In many use cases,
- * this isn't a real concern especially given the performance gain. Also, there
- * are ways to mitigate the problem further by e.g. introducing an extra
- * scheduling layer on cgroup delegation boundaries.
- *
- * The scheduler first picks the cgroup to run and then schedule the tasks
- * within by using nested weighted vtime scheduling by default. The
- * cgroup-internal scheduling can be switched to FIFO with the -f option.
- */
-#include "scx_common.bpf.h"
-#include "user_exit_info.h"
-#include "scx_example_flatcg.h"
-
-char _license[] SEC("license") = "GPL";
-
-const volatile u32 nr_cpus = 32;	/* !0 for veristat, set during init */
-const volatile u64 cgrp_slice_ns = SCX_SLICE_DFL;
-const volatile bool fifo_sched;
-const volatile bool switch_partial;
-
-u64 cvtime_now;
-struct user_exit_info uei;
-
-struct {
-	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
-	__type(key, u32);
-	__type(value, u64);
-	__uint(max_entries, FCG_NR_STATS);
-} stats SEC(".maps");
-
-static void stat_inc(enum fcg_stat_idx idx)
-{
-	u32 idx_v = idx;
-
-	u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v);
-	if (cnt_p)
-		(*cnt_p)++;
-}
-
-struct fcg_cpu_ctx {
-	u64			cur_cgid;
-	u64			cur_at;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
-	__type(key, u32);
-	__type(value, struct fcg_cpu_ctx);
-	__uint(max_entries, 1);
-} cpu_ctx SEC(".maps");
-
-struct {
-	__uint(type, BPF_MAP_TYPE_CGRP_STORAGE);
-	__uint(map_flags, BPF_F_NO_PREALLOC);
-	__type(key, int);
-	__type(value, struct fcg_cgrp_ctx);
-} cgrp_ctx SEC(".maps");
-
-struct cgv_node {
-	struct bpf_rb_node	rb_node;
-	__u64			cvtime;
-	__u64			cgid;
-};
-
-private(CGV_TREE) struct bpf_spin_lock cgv_tree_lock;
-private(CGV_TREE) struct bpf_rb_root cgv_tree __contains(cgv_node, rb_node);
-
-struct cgv_node_stash {
-	struct cgv_node __kptr *node;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__uint(max_entries, 16384);
-	__type(key, __u64);
-	__type(value, struct cgv_node_stash);
-} cgv_node_stash SEC(".maps");
-
-struct fcg_task_ctx {
-	u64		bypassed_at;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
-	__uint(map_flags, BPF_F_NO_PREALLOC);
-	__type(key, int);
-	__type(value, struct fcg_task_ctx);
-} task_ctx SEC(".maps");
-
-/* gets inc'd on weight tree changes to expire the cached hweights */
-unsigned long hweight_gen = 1;
-
-static u64 div_round_up(u64 dividend, u64 divisor)
-{
-	return (dividend + divisor - 1) / divisor;
-}
-
-static bool vtime_before(u64 a, u64 b)
-{
-	return (s64)(a - b) < 0;
-}
-
-static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
-{
-	struct cgv_node *cgc_a, *cgc_b;
-
-	cgc_a = container_of(a, struct cgv_node, rb_node);
-	cgc_b = container_of(b, struct cgv_node, rb_node);
-
-	return cgc_a->cvtime < cgc_b->cvtime;
-}
-
-static struct fcg_cpu_ctx *find_cpu_ctx(void)
-{
-	struct fcg_cpu_ctx *cpuc;
-	u32 idx = 0;
-
-	cpuc = bpf_map_lookup_elem(&cpu_ctx, &idx);
-	if (!cpuc) {
-		scx_bpf_error("cpu_ctx lookup failed");
-		return NULL;
-	}
-	return cpuc;
-}
-
-static struct fcg_cgrp_ctx *find_cgrp_ctx(struct cgroup *cgrp)
-{
-	struct fcg_cgrp_ctx *cgc;
-
-	cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0);
-	if (!cgc) {
-		scx_bpf_error("cgrp_ctx lookup failed for cgid %llu", cgrp->kn->id);
-		return NULL;
-	}
-	return cgc;
-}
-
-static struct fcg_cgrp_ctx *find_ancestor_cgrp_ctx(struct cgroup *cgrp, int level)
-{
-	struct fcg_cgrp_ctx *cgc;
-
-	cgrp = bpf_cgroup_ancestor(cgrp, level);
-	if (!cgrp) {
-		scx_bpf_error("ancestor cgroup lookup failed");
-		return NULL;
-	}
-
-	cgc = find_cgrp_ctx(cgrp);
-	if (!cgc)
-		scx_bpf_error("ancestor cgrp_ctx lookup failed");
-	bpf_cgroup_release(cgrp);
-	return cgc;
-}
-
-static void cgrp_refresh_hweight(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc)
-{
-	int level;
-
-	if (!cgc->nr_active) {
-		stat_inc(FCG_STAT_HWT_SKIP);
-		return;
-	}
-
-	if (cgc->hweight_gen == hweight_gen) {
-		stat_inc(FCG_STAT_HWT_CACHE);
-		return;
-	}
-
-	stat_inc(FCG_STAT_HWT_UPDATES);
-	bpf_for(level, 0, cgrp->level + 1) {
-		struct fcg_cgrp_ctx *cgc;
-		bool is_active;
-
-		cgc = find_ancestor_cgrp_ctx(cgrp, level);
-		if (!cgc)
-			break;
-
-		if (!level) {
-			cgc->hweight = FCG_HWEIGHT_ONE;
-			cgc->hweight_gen = hweight_gen;
-		} else {
-			struct fcg_cgrp_ctx *pcgc;
-
-			pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1);
-			if (!pcgc)
-				break;
-
-			/*
-			 * We can be oppotunistic here and not grab the
-			 * cgv_tree_lock and deal with the occasional races.
-			 * However, hweight updates are already cached and
-			 * relatively low-frequency. Let's just do the
-			 * straightforward thing.
-			 */
-			bpf_spin_lock(&cgv_tree_lock);
-			is_active = cgc->nr_active;
-			if (is_active) {
-				cgc->hweight_gen = pcgc->hweight_gen;
-				cgc->hweight =
-					div_round_up(pcgc->hweight * cgc->weight,
-						     pcgc->child_weight_sum);
-			}
-			bpf_spin_unlock(&cgv_tree_lock);
-
-			if (!is_active) {
-				stat_inc(FCG_STAT_HWT_RACE);
-				break;
-			}
-		}
-	}
-}
-
-static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc)
-{
-	u64 delta, cvtime, max_budget;
-
-	/*
-	 * A node which is on the rbtree can't be pointed to from elsewhere yet
-	 * and thus can't be updated and repositioned. Instead, we collect the
-	 * vtime deltas separately and apply it asynchronously here.
-	 */
-	delta = cgc->cvtime_delta;
-	__sync_fetch_and_sub(&cgc->cvtime_delta, delta);
-	cvtime = cgv_node->cvtime + delta;
-
-	/*
-	 * Allow a cgroup to carry the maximum budget proportional to its
-	 * hweight such that a full-hweight cgroup can immediately take up half
-	 * of the CPUs at the most while staying at the front of the rbtree.
-	 */
-	max_budget = (cgrp_slice_ns * nr_cpus * cgc->hweight) /
-		(2 * FCG_HWEIGHT_ONE);
-	if (vtime_before(cvtime, cvtime_now - max_budget))
-		cvtime = cvtime_now - max_budget;
-
-	cgv_node->cvtime = cvtime;
-}
-
-static void cgrp_enqueued(struct cgroup *cgrp, struct fcg_cgrp_ctx *cgc)
-{
-	struct cgv_node_stash *stash;
-	struct cgv_node *cgv_node;
-	u64 cgid = cgrp->kn->id;
-
-	/* paired with cmpxchg in try_pick_next_cgroup() */
-	if (__sync_val_compare_and_swap(&cgc->queued, 0, 1)) {
-		stat_inc(FCG_STAT_ENQ_SKIP);
-		return;
-	}
-
-	stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid);
-	if (!stash) {
-		scx_bpf_error("cgv_node lookup failed for cgid %llu", cgid);
-		return;
-	}
-
-	/* NULL if the node is already on the rbtree */
-	cgv_node = bpf_kptr_xchg(&stash->node, NULL);
-	if (!cgv_node) {
-		stat_inc(FCG_STAT_ENQ_RACE);
-		return;
-	}
-
-	bpf_spin_lock(&cgv_tree_lock);
-	cgrp_cap_budget(cgv_node, cgc);
-	bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less);
-	bpf_spin_unlock(&cgv_tree_lock);
-}
-
-void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags)
-{
-	struct fcg_task_ctx *taskc;
-	struct cgroup *cgrp;
-	struct fcg_cgrp_ctx *cgc;
-
-	taskc = bpf_task_storage_get(&task_ctx, p, 0, 0);
-	if (!taskc) {
-		scx_bpf_error("task_ctx lookup failed");
-		return;
-	}
-
-	/*
-	 * If select_cpu_dfl() is recommending local enqueue, the target CPU is
-	 * idle. Follow it and charge the cgroup later in fcg_stopping() after
-	 * the fact. Use the same mechanism to deal with tasks with custom
-	 * affinities so that we don't have to worry about per-cgroup dq's
-	 * containing tasks that can't be executed from some CPUs.
-	 */
-	if ((enq_flags & SCX_ENQ_LOCAL) || p->nr_cpus_allowed != nr_cpus) {
-		/*
-		 * Tell fcg_stopping() that this bypassed the regular scheduling
-		 * path and should be force charged to the cgroup. 0 is used to
-		 * indicate that the task isn't bypassing, so if the current
-		 * runtime is 0, go back by one nanosecond.
-		 */
-		taskc->bypassed_at = p->se.sum_exec_runtime ?: (u64)-1;
-
-		/*
-		 * The global dq is deprioritized as we don't want to let tasks
-		 * to boost themselves by constraining its cpumask. The
-		 * deprioritization is rather severe, so let's not apply that to
-		 * per-cpu kernel threads. This is ham-fisted. We probably wanna
-		 * implement per-cgroup fallback dq's instead so that we have
-		 * more control over when tasks with custom cpumask get issued.
-		 */
-		if ((enq_flags & SCX_ENQ_LOCAL) ||
-		    (p->nr_cpus_allowed == 1 && (p->flags & PF_KTHREAD))) {
-			stat_inc(FCG_STAT_LOCAL);
-			scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
-		} else {
-			stat_inc(FCG_STAT_GLOBAL);
-			scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
-		}
-		return;
-	}
-
-	cgrp = scx_bpf_task_cgroup(p);
-	cgc = find_cgrp_ctx(cgrp);
-	if (!cgc)
-		goto out_release;
-
-	if (fifo_sched) {
-		scx_bpf_dispatch(p, cgrp->kn->id, SCX_SLICE_DFL, enq_flags);
-	} else {
-		u64 tvtime = p->scx.dsq_vtime;
-
-		/*
-		 * Limit the amount of budget that an idling task can accumulate
-		 * to one slice.
-		 */
-		if (vtime_before(tvtime, cgc->tvtime_now - SCX_SLICE_DFL))
-			tvtime = cgc->tvtime_now - SCX_SLICE_DFL;
-
-		scx_bpf_dispatch_vtime(p, cgrp->kn->id, SCX_SLICE_DFL,
-				       tvtime, enq_flags);
-	}
-
-	cgrp_enqueued(cgrp, cgc);
-out_release:
-	bpf_cgroup_release(cgrp);
-}
-
-/*
- * Walk the cgroup tree to update the active weight sums as tasks wake up and
- * sleep. The weight sums are used as the base when calculating the proportion a
- * given cgroup or task is entitled to at each level.
- */
-static void update_active_weight_sums(struct cgroup *cgrp, bool runnable)
-{
-	struct fcg_cgrp_ctx *cgc;
-	bool updated = false;
-	int idx;
-
-	cgc = find_cgrp_ctx(cgrp);
-	if (!cgc)
-		return;
-
-	/*
-	 * In most cases, a hot cgroup would have multiple threads going to
-	 * sleep and waking up while the whole cgroup stays active. In leaf
-	 * cgroups, ->nr_runnable which is updated with __sync operations gates
-	 * ->nr_active updates, so that we don't have to grab the cgv_tree_lock
-	 * repeatedly for a busy cgroup which is staying active.
-	 */
-	if (runnable) {
-		if (__sync_fetch_and_add(&cgc->nr_runnable, 1))
-			return;
-		stat_inc(FCG_STAT_ACT);
-	} else {
-		if (__sync_sub_and_fetch(&cgc->nr_runnable, 1))
-			return;
-		stat_inc(FCG_STAT_DEACT);
-	}
-
-	/*
-	 * If @cgrp is becoming runnable, its hweight should be refreshed after
-	 * it's added to the weight tree so that enqueue has the up-to-date
-	 * value. If @cgrp is becoming quiescent, the hweight should be
-	 * refreshed before it's removed from the weight tree so that the usage
-	 * charging which happens afterwards has access to the latest value.
-	 */
-	if (!runnable)
-		cgrp_refresh_hweight(cgrp, cgc);
-
-	/* propagate upwards */
-	bpf_for(idx, 0, cgrp->level) {
-		int level = cgrp->level - idx;
-		struct fcg_cgrp_ctx *cgc, *pcgc = NULL;
-		bool propagate = false;
-
-		cgc = find_ancestor_cgrp_ctx(cgrp, level);
-		if (!cgc)
-			break;
-		if (level) {
-			pcgc = find_ancestor_cgrp_ctx(cgrp, level - 1);
-			if (!pcgc)
-				break;
-		}
-
-		/*
-		 * We need the propagation protected by a lock to synchronize
-		 * against weight changes. There's no reason to drop the lock at
-		 * each level but bpf_spin_lock() doesn't want any function
-		 * calls while locked.
-		 */
-		bpf_spin_lock(&cgv_tree_lock);
-
-		if (runnable) {
-			if (!cgc->nr_active++) {
-				updated = true;
-				if (pcgc) {
-					propagate = true;
-					pcgc->child_weight_sum += cgc->weight;
-				}
-			}
-		} else {
-			if (!--cgc->nr_active) {
-				updated = true;
-				if (pcgc) {
-					propagate = true;
-					pcgc->child_weight_sum -= cgc->weight;
-				}
-			}
-		}
-
-		bpf_spin_unlock(&cgv_tree_lock);
-
-		if (!propagate)
-			break;
-	}
-
-	if (updated)
-		__sync_fetch_and_add(&hweight_gen, 1);
-
-	if (runnable)
-		cgrp_refresh_hweight(cgrp, cgc);
-}
-
-void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags)
-{
-	struct cgroup *cgrp;
-
-	cgrp = scx_bpf_task_cgroup(p);
-	update_active_weight_sums(cgrp, true);
-	bpf_cgroup_release(cgrp);
-}
-
-void BPF_STRUCT_OPS(fcg_running, struct task_struct *p)
-{
-	struct cgroup *cgrp;
-	struct fcg_cgrp_ctx *cgc;
-
-	if (fifo_sched)
-		return;
-
-	cgrp = scx_bpf_task_cgroup(p);
-	cgc = find_cgrp_ctx(cgrp);
-	if (cgc) {
-		/*
-		 * @cgc->tvtime_now always progresses forward as tasks start
-		 * executing. The test and update can be performed concurrently
-		 * from multiple CPUs and thus racy. Any error should be
-		 * contained and temporary. Let's just live with it.
-		 */
-		if (vtime_before(cgc->tvtime_now, p->scx.dsq_vtime))
-			cgc->tvtime_now = p->scx.dsq_vtime;
-	}
-	bpf_cgroup_release(cgrp);
-}
-
-void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable)
-{
-	struct fcg_task_ctx *taskc;
-	struct cgroup *cgrp;
-	struct fcg_cgrp_ctx *cgc;
-
-	/* scale the execution time by the inverse of the weight and charge */
-	if (!fifo_sched)
-		p->scx.dsq_vtime +=
-			(SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight;
-
-	taskc = bpf_task_storage_get(&task_ctx, p, 0, 0);
-	if (!taskc) {
-		scx_bpf_error("task_ctx lookup failed");
-		return;
-	}
-
-	if (!taskc->bypassed_at)
-		return;
-
-	cgrp = scx_bpf_task_cgroup(p);
-	cgc = find_cgrp_ctx(cgrp);
-	if (cgc) {
-		__sync_fetch_and_add(&cgc->cvtime_delta,
-				     p->se.sum_exec_runtime - taskc->bypassed_at);
-		taskc->bypassed_at = 0;
-	}
-	bpf_cgroup_release(cgrp);
-}
-
-void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags)
-{
-	struct cgroup *cgrp;
-
-	cgrp = scx_bpf_task_cgroup(p);
-	update_active_weight_sums(cgrp, false);
-	bpf_cgroup_release(cgrp);
-}
-
-void BPF_STRUCT_OPS(fcg_cgroup_set_weight, struct cgroup *cgrp, u32 weight)
-{
-	struct fcg_cgrp_ctx *cgc, *pcgc = NULL;
-
-	cgc = find_cgrp_ctx(cgrp);
-	if (!cgc)
-		return;
-
-	if (cgrp->level) {
-		pcgc = find_ancestor_cgrp_ctx(cgrp, cgrp->level - 1);
-		if (!pcgc)
-			return;
-	}
-
-	bpf_spin_lock(&cgv_tree_lock);
-	if (pcgc && cgc->nr_active)
-		pcgc->child_weight_sum += (s64)weight - cgc->weight;
-	cgc->weight = weight;
-	bpf_spin_unlock(&cgv_tree_lock);
-}
-
-static bool try_pick_next_cgroup(u64 *cgidp)
-{
-	struct bpf_rb_node *rb_node;
-	struct cgv_node_stash *stash;
-	struct cgv_node *cgv_node;
-	struct fcg_cgrp_ctx *cgc;
-	struct cgroup *cgrp;
-	u64 cgid;
-
-	/* pop the front cgroup and wind cvtime_now accordingly */
-	bpf_spin_lock(&cgv_tree_lock);
-
-	rb_node = bpf_rbtree_first(&cgv_tree);
-	if (!rb_node) {
-		bpf_spin_unlock(&cgv_tree_lock);
-		stat_inc(FCG_STAT_PNC_NO_CGRP);
-		*cgidp = 0;
-		return true;
-	}
-
-	rb_node = bpf_rbtree_remove(&cgv_tree, rb_node);
-	bpf_spin_unlock(&cgv_tree_lock);
-
-	cgv_node = container_of(rb_node, struct cgv_node, rb_node);
-	cgid = cgv_node->cgid;
-
-	if (vtime_before(cvtime_now, cgv_node->cvtime))
-		cvtime_now = cgv_node->cvtime;
-
-	/*
-	 * If lookup fails, the cgroup's gone. Free and move on. See
-	 * fcg_cgroup_exit().
-	 */
-	cgrp = bpf_cgroup_from_id(cgid);
-	if (!cgrp) {
-		stat_inc(FCG_STAT_PNC_GONE);
-		goto out_free;
-	}
-
-	cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0);
-	if (!cgc) {
-		bpf_cgroup_release(cgrp);
-		stat_inc(FCG_STAT_PNC_GONE);
-		goto out_free;
-	}
-
-	if (!scx_bpf_consume(cgid)) {
-		bpf_cgroup_release(cgrp);
-		stat_inc(FCG_STAT_PNC_EMPTY);
-		goto out_stash;
-	}
-
-	/*
-	 * Successfully consumed from the cgroup. This will be our current
-	 * cgroup for the new slice. Refresh its hweight.
-	 */
-	cgrp_refresh_hweight(cgrp, cgc);
-
-	bpf_cgroup_release(cgrp);
-
-	/*
-	 * As the cgroup may have more tasks, add it back to the rbtree. Note
-	 * that here we charge the full slice upfront and then exact later
-	 * according to the actual consumption. This prevents lowpri thundering
-	 * herd from saturating the machine.
-	 */
-	bpf_spin_lock(&cgv_tree_lock);
-	cgv_node->cvtime += cgrp_slice_ns * FCG_HWEIGHT_ONE / (cgc->hweight ?: 1);
-	cgrp_cap_budget(cgv_node, cgc);
-	bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less);
-	bpf_spin_unlock(&cgv_tree_lock);
-
-	*cgidp = cgid;
-	stat_inc(FCG_STAT_PNC_NEXT);
-	return true;
-
-out_stash:
-	stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid);
-	if (!stash) {
-		stat_inc(FCG_STAT_PNC_GONE);
-		goto out_free;
-	}
-
-	/*
-	 * Paired with cmpxchg in cgrp_enqueued(). If they see the following
-	 * transition, they'll enqueue the cgroup. If they are earlier, we'll
-	 * see their task in the dq below and requeue the cgroup.
-	 */
-	__sync_val_compare_and_swap(&cgc->queued, 1, 0);
-
-	if (scx_bpf_dsq_nr_queued(cgid)) {
-		bpf_spin_lock(&cgv_tree_lock);
-		bpf_rbtree_add(&cgv_tree, &cgv_node->rb_node, cgv_node_less);
-		bpf_spin_unlock(&cgv_tree_lock);
-	} else {
-		cgv_node = bpf_kptr_xchg(&stash->node, cgv_node);
-		if (cgv_node) {
-			scx_bpf_error("unexpected !NULL cgv_node stash");
-			goto out_free;
-		}
-	}
-
-	return false;
-
-out_free:
-	bpf_obj_drop(cgv_node);
-	return false;
-}
-
-void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev)
-{
-	struct fcg_cpu_ctx *cpuc;
-	struct fcg_cgrp_ctx *cgc;
-	struct cgroup *cgrp;
-	u64 now = bpf_ktime_get_ns();
-
-	cpuc = find_cpu_ctx();
-	if (!cpuc)
-		return;
-
-	if (!cpuc->cur_cgid)
-		goto pick_next_cgroup;
-
-	if (vtime_before(now, cpuc->cur_at + cgrp_slice_ns)) {
-		if (scx_bpf_consume(cpuc->cur_cgid)) {
-			stat_inc(FCG_STAT_CNS_KEEP);
-			return;
-		}
-		stat_inc(FCG_STAT_CNS_EMPTY);
-	} else {
-		stat_inc(FCG_STAT_CNS_EXPIRE);
-	}
-
-	/*
-	 * The current cgroup is expiring. It was already charged a full slice.
-	 * Calculate the actual usage and accumulate the delta.
-	 */
-	cgrp = bpf_cgroup_from_id(cpuc->cur_cgid);
-	if (!cgrp) {
-		stat_inc(FCG_STAT_CNS_GONE);
-		goto pick_next_cgroup;
-	}
-
-	cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0, 0);
-	if (cgc) {
-		/*
-		 * We want to update the vtime delta and then look for the next
-		 * cgroup to execute but the latter needs to be done in a loop
-		 * and we can't keep the lock held. Oh well...
-		 */
-		bpf_spin_lock(&cgv_tree_lock);
-		__sync_fetch_and_add(&cgc->cvtime_delta,
-				     (cpuc->cur_at + cgrp_slice_ns - now) *
-				     FCG_HWEIGHT_ONE / (cgc->hweight ?: 1));
-		bpf_spin_unlock(&cgv_tree_lock);
-	} else {
-		stat_inc(FCG_STAT_CNS_GONE);
-	}
-
-	bpf_cgroup_release(cgrp);
-
-pick_next_cgroup:
-	cpuc->cur_at = now;
-
-	if (scx_bpf_consume(SCX_DSQ_GLOBAL)) {
-		cpuc->cur_cgid = 0;
-		return;
-	}
-
-	bpf_repeat(BPF_MAX_LOOPS) {
-		if (try_pick_next_cgroup(&cpuc->cur_cgid))
-			break;
-	}
-}
-
-s32 BPF_STRUCT_OPS(fcg_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
-{
-	struct fcg_task_ctx *taskc;
-
-	/*
-	 * @p is new. Let's ensure that its task_ctx is available. We can sleep
-	 * in this function and the following will automatically use GFP_KERNEL.
-	 */
-	taskc = bpf_task_storage_get(&task_ctx, p, 0,
-				     BPF_LOCAL_STORAGE_GET_F_CREATE);
-	if (!taskc)
-		return -ENOMEM;
-
-	taskc->bypassed_at = 0;
-	return 0;
-}
-
-int BPF_STRUCT_OPS_SLEEPABLE(fcg_cgroup_init, struct cgroup *cgrp,
-			     struct scx_cgroup_init_args *args)
-{
-	struct fcg_cgrp_ctx *cgc;
-	struct cgv_node *cgv_node;
-	struct cgv_node_stash empty_stash = {}, *stash;
-	u64 cgid = cgrp->kn->id;
-	int ret;
-
-	/*
-	 * Technically incorrect as cgroup ID is full 64bit while dq ID is
-	 * 63bit. Should not be a problem in practice and easy to spot in the
-	 * unlikely case that it breaks.
-	 */
-	ret = scx_bpf_create_dsq(cgid, -1);
-	if (ret)
-		return ret;
-
-	cgc = bpf_cgrp_storage_get(&cgrp_ctx, cgrp, 0,
-				   BPF_LOCAL_STORAGE_GET_F_CREATE);
-	if (!cgc) {
-		ret = -ENOMEM;
-		goto err_destroy_dsq;
-	}
-
-	cgc->weight = args->weight;
-	cgc->hweight = FCG_HWEIGHT_ONE;
-
-	ret = bpf_map_update_elem(&cgv_node_stash, &cgid, &empty_stash,
-				  BPF_NOEXIST);
-	if (ret) {
-		if (ret != -ENOMEM)
-			scx_bpf_error("unexpected stash creation error (%d)",
-				      ret);
-		goto err_destroy_dsq;
-	}
-
-	stash = bpf_map_lookup_elem(&cgv_node_stash, &cgid);
-	if (!stash) {
-		scx_bpf_error("unexpected cgv_node stash lookup failure");
-		ret = -ENOENT;
-		goto err_destroy_dsq;
-	}
-
-	cgv_node = bpf_obj_new(struct cgv_node);
-	if (!cgv_node) {
-		ret = -ENOMEM;
-		goto err_del_cgv_node;
-	}
-
-	cgv_node->cgid = cgid;
-	cgv_node->cvtime = cvtime_now;
-
-	cgv_node = bpf_kptr_xchg(&stash->node, cgv_node);
-	if (cgv_node) {
-		scx_bpf_error("unexpected !NULL cgv_node stash");
-		ret = -EBUSY;
-		goto err_drop;
-	}
-
-	return 0;
-
-err_drop:
-	bpf_obj_drop(cgv_node);
-err_del_cgv_node:
-	bpf_map_delete_elem(&cgv_node_stash, &cgid);
-err_destroy_dsq:
-	scx_bpf_destroy_dsq(cgid);
-	return ret;
-}
-
-void BPF_STRUCT_OPS(fcg_cgroup_exit, struct cgroup *cgrp)
-{
-	u64 cgid = cgrp->kn->id;
-
-	/*
-	 * For now, there's no way find and remove the cgv_node if it's on the
-	 * cgv_tree. Let's drain them in the dispatch path as they get popped
-	 * off the front of the tree.
-	 */
-	bpf_map_delete_elem(&cgv_node_stash, &cgid);
-	scx_bpf_destroy_dsq(cgid);
-}
-
-s32 BPF_STRUCT_OPS(fcg_init)
-{
-	if (!switch_partial)
-		scx_bpf_switch_all();
-	return 0;
-}
-
-void BPF_STRUCT_OPS(fcg_exit, struct scx_exit_info *ei)
-{
-	uei_record(&uei, ei);
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops flatcg_ops = {
-	.enqueue		= (void *)fcg_enqueue,
-	.dispatch		= (void *)fcg_dispatch,
-	.runnable		= (void *)fcg_runnable,
-	.running		= (void *)fcg_running,
-	.stopping		= (void *)fcg_stopping,
-	.quiescent		= (void *)fcg_quiescent,
-	.prep_enable		= (void *)fcg_prep_enable,
-	.cgroup_set_weight	= (void *)fcg_cgroup_set_weight,
-	.cgroup_init		= (void *)fcg_cgroup_init,
-	.cgroup_exit		= (void *)fcg_cgroup_exit,
-	.init			= (void *)fcg_init,
-	.exit			= (void *)fcg_exit,
-	.flags			= SCX_OPS_CGROUP_KNOB_WEIGHT | SCX_OPS_ENQ_EXITING,
-	.name			= "flatcg",
-};
diff --git a/tools/sched_ext/scx_example_flatcg.c b/tools/sched_ext/scx_example_flatcg.c
deleted file mode 100644
index f9c8a5b84a70..000000000000
--- a/tools/sched_ext/scx_example_flatcg.c
+++ /dev/null
@@ -1,232 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2023 David Vernet <dvernet@meta.com>
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <signal.h>
-#include <unistd.h>
-#include <limits.h>
-#include <fcntl.h>
-#include <time.h>
-#include <assert.h>
-#include <bpf/bpf.h>
-#include "user_exit_info.h"
-#include "scx_example_flatcg.h"
-#include "scx_example_flatcg.skel.h"
-
-#ifndef FILEID_KERNFS
-#define FILEID_KERNFS		0xfe
-#endif
-
-const char help_fmt[] =
-"A flattened cgroup hierarchy sched_ext scheduler.\n"
-"\n"
-"See the top-level comment in .bpf.c for more details.\n"
-"\n"
-"Usage: %s [-s SLICE_US] [-i INTERVAL] [-f] [-p]\n"
-"\n"
-"  -s SLICE_US   Override slice duration\n"
-"  -i INTERVAL   Report interval\n"
-"  -f            Use FIFO scheduling instead of weighted vtime scheduling\n"
-"  -p            Switch only tasks on SCHED_EXT policy intead of all\n"
-"  -h            Display this help and exit\n";
-
-static volatile int exit_req;
-
-static void sigint_handler(int dummy)
-{
-	exit_req = 1;
-}
-
-static float read_cpu_util(__u64 *last_sum, __u64 *last_idle)
-{
-	FILE *fp;
-	char buf[4096];
-	char *line, *cur = NULL, *tok;
-	__u64 sum = 0, idle = 0;
-	__u64 delta_sum, delta_idle;
-	int idx;
-
-	fp = fopen("/proc/stat", "r");
-	if (!fp) {
-		perror("fopen(\"/proc/stat\")");
-		return 0.0;
-	}
-
-	if (!fgets(buf, sizeof(buf), fp)) {
-		perror("fgets(\"/proc/stat\")");
-		fclose(fp);
-		return 0.0;
-	}
-	fclose(fp);
-
-	line = buf;
-	for (idx = 0; (tok = strtok_r(line, " \n", &cur)); idx++) {
-		char *endp = NULL;
-		__u64 v;
-
-		if (idx == 0) {
-			line = NULL;
-			continue;
-		}
-		v = strtoull(tok, &endp, 0);
-		if (!endp || *endp != '\0') {
-			fprintf(stderr, "failed to parse %dth field of /proc/stat (\"%s\")\n",
-				idx, tok);
-			continue;
-		}
-		sum += v;
-		if (idx == 4)
-			idle = v;
-	}
-
-	delta_sum = sum - *last_sum;
-	delta_idle = idle - *last_idle;
-	*last_sum = sum;
-	*last_idle = idle;
-
-	return delta_sum ? (float)(delta_sum - delta_idle) / delta_sum : 0.0;
-}
-
-static void fcg_read_stats(struct scx_example_flatcg *skel, __u64 *stats)
-{
-	__u64 cnts[FCG_NR_STATS][skel->rodata->nr_cpus];
-	__u32 idx;
-
-	memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS);
-
-	for (idx = 0; idx < FCG_NR_STATS; idx++) {
-		int ret, cpu;
-
-		ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats),
-					  &idx, cnts[idx]);
-		if (ret < 0)
-			continue;
-		for (cpu = 0; cpu < skel->rodata->nr_cpus; cpu++)
-			stats[idx] += cnts[idx][cpu];
-	}
-}
-
-int main(int argc, char **argv)
-{
-	struct scx_example_flatcg *skel;
-	struct bpf_link *link;
-	struct timespec intv_ts = { .tv_sec = 2, .tv_nsec = 0 };
-	bool dump_cgrps = false;
-	__u64 last_cpu_sum = 0, last_cpu_idle = 0;
-	__u64 last_stats[FCG_NR_STATS] = {};
-	unsigned long seq = 0;
-	s32 opt;
-
-	signal(SIGINT, sigint_handler);
-	signal(SIGTERM, sigint_handler);
-
-	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
-
-	skel = scx_example_flatcg__open();
-	if (!skel) {
-		fprintf(stderr, "Failed to open: %s\n", strerror(errno));
-		return 1;
-	}
-
-	skel->rodata->nr_cpus = libbpf_num_possible_cpus();
-
-	while ((opt = getopt(argc, argv, "s:i:dfph")) != -1) {
-		double v;
-
-		switch (opt) {
-		case 's':
-			v = strtod(optarg, NULL);
-			skel->rodata->cgrp_slice_ns = v * 1000;
-			break;
-		case 'i':
-			v = strtod(optarg, NULL);
-			intv_ts.tv_sec = v;
-			intv_ts.tv_nsec = (v - (float)intv_ts.tv_sec) * 1000000000;
-			break;
-		case 'd':
-			dump_cgrps = true;
-			break;
-		case 'f':
-			skel->rodata->fifo_sched = true;
-			break;
-		case 'p':
-			skel->rodata->switch_partial = true;
-			break;
-		case 'h':
-		default:
-			fprintf(stderr, help_fmt, basename(argv[0]));
-			return opt != 'h';
-		}
-	}
-
-	printf("slice=%.1lfms intv=%.1lfs dump_cgrps=%d",
-	       (double)skel->rodata->cgrp_slice_ns / 1000000.0,
-	       (double)intv_ts.tv_sec + (double)intv_ts.tv_nsec / 1000000000.0,
-	       dump_cgrps);
-
-	if (scx_example_flatcg__load(skel)) {
-		fprintf(stderr, "Failed to load: %s\n", strerror(errno));
-		return 1;
-	}
-
-	link = bpf_map__attach_struct_ops(skel->maps.flatcg_ops);
-	if (!link) {
-		fprintf(stderr, "Failed to attach_struct_ops: %s\n",
-			strerror(errno));
-		return 1;
-	}
-
-	while (!exit_req && !uei_exited(&skel->bss->uei)) {
-		__u64 acc_stats[FCG_NR_STATS];
-		__u64 stats[FCG_NR_STATS];
-		float cpu_util;
-		int i;
-
-		cpu_util = read_cpu_util(&last_cpu_sum, &last_cpu_idle);
-
-		fcg_read_stats(skel, acc_stats);
-		for (i = 0; i < FCG_NR_STATS; i++)
-			stats[i] = acc_stats[i] - last_stats[i];
-
-		memcpy(last_stats, acc_stats, sizeof(acc_stats));
-
-		printf("\n[SEQ %6lu cpu=%5.1lf hweight_gen=%lu]\n",
-		       seq++, cpu_util * 100.0, skel->data->hweight_gen);
-		printf("       act:%6llu  deact:%6llu local:%6llu global:%6llu\n",
-		       stats[FCG_STAT_ACT],
-		       stats[FCG_STAT_DEACT],
-		       stats[FCG_STAT_LOCAL],
-		       stats[FCG_STAT_GLOBAL]);
-		printf("HWT   skip:%6llu   race:%6llu cache:%6llu update:%6llu\n",
-		       stats[FCG_STAT_HWT_SKIP],
-		       stats[FCG_STAT_HWT_RACE],
-		       stats[FCG_STAT_HWT_CACHE],
-		       stats[FCG_STAT_HWT_UPDATES]);
-		printf("ENQ   skip:%6llu   race:%6llu\n",
-		       stats[FCG_STAT_ENQ_SKIP],
-		       stats[FCG_STAT_ENQ_RACE]);
-		printf("CNS   keep:%6llu expire:%6llu empty:%6llu   gone:%6llu\n",
-		       stats[FCG_STAT_CNS_KEEP],
-		       stats[FCG_STAT_CNS_EXPIRE],
-		       stats[FCG_STAT_CNS_EMPTY],
-		       stats[FCG_STAT_CNS_GONE]);
-		printf("PNC nocgrp:%6llu   next:%6llu empty:%6llu   gone:%6llu\n",
-		       stats[FCG_STAT_PNC_NO_CGRP],
-		       stats[FCG_STAT_PNC_NEXT],
-		       stats[FCG_STAT_PNC_EMPTY],
-		       stats[FCG_STAT_PNC_GONE]);
-		printf("BAD remove:%6llu\n",
-		       acc_stats[FCG_STAT_BAD_REMOVAL]);
-
-		nanosleep(&intv_ts, NULL);
-	}
-
-	bpf_link__destroy(link);
-	uei_print(&skel->bss->uei);
-	scx_example_flatcg__destroy(skel);
-	return 0;
-}
diff --git a/tools/sched_ext/scx_example_flatcg.h b/tools/sched_ext/scx_example_flatcg.h
deleted file mode 100644
index 490758ed41f0..000000000000
--- a/tools/sched_ext/scx_example_flatcg.h
+++ /dev/null
@@ -1,49 +0,0 @@
-#ifndef __SCX_EXAMPLE_FLATCG_H
-#define __SCX_EXAMPLE_FLATCG_H
-
-enum {
-	FCG_HWEIGHT_ONE		= 1LLU << 16,
-};
-
-enum fcg_stat_idx {
-	FCG_STAT_ACT,
-	FCG_STAT_DEACT,
-	FCG_STAT_LOCAL,
-	FCG_STAT_GLOBAL,
-
-	FCG_STAT_HWT_UPDATES,
-	FCG_STAT_HWT_CACHE,
-	FCG_STAT_HWT_SKIP,
-	FCG_STAT_HWT_RACE,
-
-	FCG_STAT_ENQ_SKIP,
-	FCG_STAT_ENQ_RACE,
-
-	FCG_STAT_CNS_KEEP,
-	FCG_STAT_CNS_EXPIRE,
-	FCG_STAT_CNS_EMPTY,
-	FCG_STAT_CNS_GONE,
-
-	FCG_STAT_PNC_NO_CGRP,
-	FCG_STAT_PNC_NEXT,
-	FCG_STAT_PNC_EMPTY,
-	FCG_STAT_PNC_GONE,
-
-	FCG_STAT_BAD_REMOVAL,
-
-	FCG_NR_STATS,
-};
-
-struct fcg_cgrp_ctx {
-	u32			nr_active;
-	u32			nr_runnable;
-	u32			queued;
-	u32			weight;
-	u32			hweight;
-	u64			child_weight_sum;
-	u64			hweight_gen;
-	s64			cvtime_delta;
-	u64			tvtime_now;
-};
-
-#endif /* __SCX_EXAMPLE_FLATCG_H */
diff --git a/tools/sched_ext/scx_example_pair.bpf.c b/tools/sched_ext/scx_example_pair.bpf.c
deleted file mode 100644
index 279efe58b777..000000000000
--- a/tools/sched_ext/scx_example_pair.bpf.c
+++ /dev/null
@@ -1,627 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A demo sched_ext core-scheduler which always makes every sibling CPU pair
- * execute from the same CPU cgroup.
- *
- * This scheduler is a minimal implementation and would need some form of
- * priority handling both inside each cgroup and across the cgroups to be
- * practically useful.
- *
- * Each CPU in the system is paired with exactly one other CPU, according to a
- * "stride" value that can be specified when the BPF scheduler program is first
- * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee
- * that they will only ever schedule tasks that belong to the same CPU cgroup.
- *
- * Scheduler Initialization
- * ------------------------
- *
- * The scheduler BPF program is first initialized from user space, before it is
- * enabled. During this initialization process, each CPU on the system is
- * assigned several values that are constant throughout its runtime:
- *
- * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling
- *		  decisions. Paired CPUs always schedule tasks from the same
- *		  CPU cgroup, and synchronize with each other to guarantee
- *		  that this constraint is not violated.
- * 2. *Pair ID*:  Each CPU pair is assigned a Pair ID, which is used to access
- *		  a struct pair_ctx object that is shared between the pair.
- * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the
- *		       pair. Each struct pair_ctx has an active_mask field,
- *		       which is a bitmap used to indicate whether each core
- *		       in the pair currently has an actively running task.
- *		       This index specifies which entry in the bitmap corresponds
- *		       to each CPU in the pair.
- *
- * During this initialization, the CPUs are paired according to a "stride" that
- * may be specified when invoking the user space program that initializes and
- * loads the scheduler. By default, the stride is 1/2 the total number of CPUs.
- *
- * Tasks and cgroups
- * -----------------
- *
- * Every cgroup in the system is registered with the scheduler using the
- * pair_cgroup_init() callback, and every task in the system is associated with
- * exactly one cgroup. At a high level, the idea with the pair scheduler is to
- * always schedule tasks from the same cgroup within a given CPU pair. When a
- * task is enqueued (i.e. passed to the pair_enqueue() callback function), its
- * cgroup ID is read from its task struct, and then a corresponding queue map
- * is used to FIFO-enqueue the task for that cgroup.
- *
- * If you look through the implementation of the scheduler, you'll notice that
- * there is quite a bit of complexity involved with looking up the per-cgroup
- * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash
- * BPF hash map that is used to map a cgroup ID to a globally unique ID that's
- * allocated in the BPF program. This is done because we use separate maps to
- * store the FIFO queue of tasks, and the length of that map, per cgroup. This
- * complexity is only present because of current deficiencies in BPF that will
- * soon be addressed. The main point to keep in mind is that newly enqueued
- * tasks are added to their cgroup's FIFO queue.
- *
- * Dispatching tasks
- * -----------------
- *
- * This section will describe how enqueued tasks are dispatched and scheduled.
- * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is
- * as follows:
- *
- * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is
- *    the structure that's used to synchronize amongst the two pair CPUs in their
- *    scheduling decisions. After any of the following events have occurred:
- *
- * - The cgroup's slice run has expired, or
- * - The cgroup becomes empty, or
- * - Either CPU in the pair is preempted by a higher priority scheduling class
- *
- * The cgroup transitions to the draining state and stops executing new tasks
- * from the cgroup.
- *
- * 2. If the pair is still executing a task, mark the pair_ctx as draining, and
- *    wait for the pair CPU to be preempted.
- *
- * 3. Otherwise, if the pair CPU is not running a task, we can move onto
- *    scheduling new tasks. Pop the next cgroup id from the top_q queue.
- *
- * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it.
- *
- * Note again that this scheduling behavior is simple, but the implementation
- * is complex mostly because this it hits several BPF shortcomings and has to
- * work around in often awkward ways. Most of the shortcomings are expected to
- * be resolved in the near future which should allow greatly simplifying this
- * scheduler.
- *
- * Dealing with preemption
- * -----------------------
- *
- * SCX is the lowest priority sched_class, and could be preempted by them at
- * any time. To address this, the scheduler implements pair_cpu_release() and
- * pair_cpu_acquire() callbacks which are invoked by the core scheduler when
- * the scheduler loses and gains control of the CPU respectively.
- *
- * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and
- * then invoke:
- *
- * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT);
- *
- * This preempts the pair CPU, and waits until it has re-entered the scheduler
- * before returning. This is necessary to ensure that the higher priority
- * sched_class that preempted our scheduler does not schedule a task
- * concurrently with our pair CPU.
- *
- * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption
- * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable
- * pair scheduling.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#include "scx_common.bpf.h"
-#include "scx_example_pair.h"
-
-char _license[] SEC("license") = "GPL";
-
-const volatile bool switch_partial;
-
-/* !0 for veristat, set during init */
-const volatile u32 nr_cpu_ids = 64;
-
-/* a pair of CPUs stay on a cgroup for this duration */
-const volatile u32 pair_batch_dur_ns = SCX_SLICE_DFL;
-
-/* cpu ID -> pair cpu ID */
-const volatile s32 pair_cpu[MAX_CPUS] = { [0 ... MAX_CPUS - 1] = -1 };
-
-/* cpu ID -> pair_id */
-const volatile u32 pair_id[MAX_CPUS];
-
-/* CPU ID -> CPU # in the pair (0 or 1) */
-const volatile u32 in_pair_idx[MAX_CPUS];
-
-struct pair_ctx {
-	struct bpf_spin_lock	lock;
-
-	/* the cgroup the pair is currently executing */
-	u64			cgid;
-
-	/* the pair started executing the current cgroup at */
-	u64			started_at;
-
-	/* whether the current cgroup is draining */
-	bool			draining;
-
-	/* the CPUs that are currently active on the cgroup */
-	u32			active_mask;
-
-	/*
-	 * the CPUs that are currently preempted and running tasks in a
-	 * different scheduler.
-	 */
-	u32			preempted_mask;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_ARRAY);
-	__uint(max_entries, MAX_CPUS / 2);
-	__type(key, u32);
-	__type(value, struct pair_ctx);
-} pair_ctx SEC(".maps");
-
-/* queue of cgrp_q's possibly with tasks on them */
-struct {
-	__uint(type, BPF_MAP_TYPE_QUEUE);
-	/*
-	 * Because it's difficult to build strong synchronization encompassing
-	 * multiple non-trivial operations in BPF, this queue is managed in an
-	 * opportunistic way so that we guarantee that a cgroup w/ active tasks
-	 * is always on it but possibly multiple times. Once we have more robust
-	 * synchronization constructs and e.g. linked list, we should be able to
-	 * do this in a prettier way but for now just size it big enough.
-	 */
-	__uint(max_entries, 4 * MAX_CGRPS);
-	__type(value, u64);
-} top_q SEC(".maps");
-
-/* per-cgroup q which FIFOs the tasks from the cgroup */
-struct cgrp_q {
-	__uint(type, BPF_MAP_TYPE_QUEUE);
-	__uint(max_entries, MAX_QUEUED);
-	__type(value, u32);
-};
-
-/*
- * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local
- * storage; however, a cgroup local storage can only be accessed from the BPF
- * progs attached to the cgroup. For now, work around by allocating array of
- * cgrp_q's and then allocating per-cgroup indices.
- *
- * Another caveat: It's difficult to populate a large array of maps statically
- * or from BPF. Initialize it from userland.
- */
-struct {
-	__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
-	__uint(max_entries, MAX_CGRPS);
-	__type(key, s32);
-	__array(values, struct cgrp_q);
-} cgrp_q_arr SEC(".maps");
-
-static u64 cgrp_q_len[MAX_CGRPS];
-
-/*
- * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be
- * useful to have as a map type.
- */
-static u32 cgrp_q_idx_cursor;
-static u64 cgrp_q_idx_busy[MAX_CGRPS];
-
-/*
- * All added up, the following is what we do:
- *
- * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking
- *    for a free ID. If not found, fail cgroup creation with -EBUSY.
- *
- * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following
- *    cgrp_q_idx_hash.
- *
- * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from
- *    cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr.
- *
- * This is sadly complicated for something pretty simple. Hopefully, we should
- * be able to simplify in the future.
- */
-struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__uint(max_entries, MAX_CGRPS);
-	__uint(key_size, sizeof(u64));		/* cgrp ID */
-	__uint(value_size, sizeof(s32));	/* cgrp_q idx */
-} cgrp_q_idx_hash SEC(".maps");
-
-/* statistics */
-u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions;
-u64 nr_exps, nr_exp_waits, nr_exp_empty;
-u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty;
-
-struct user_exit_info uei;
-
-static bool time_before(u64 a, u64 b)
-{
-	return (s64)(a - b) < 0;
-}
-
-void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags)
-{
-	struct cgroup *cgrp;
-	struct cgrp_q *cgq;
-	s32 pid = p->pid;
-	u64 cgid;
-	u32 *q_idx;
-	u64 *cgq_len;
-
-	__sync_fetch_and_add(&nr_total, 1);
-
-	cgrp = scx_bpf_task_cgroup(p);
-	cgid = cgrp->kn->id;
-	bpf_cgroup_release(cgrp);
-
-	/* find the cgroup's q and push @p into it */
-	q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid);
-	if (!q_idx) {
-		scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid);
-		return;
-	}
-
-	cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx);
-	if (!cgq) {
-		scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]",
-			      cgid, *q_idx);
-		return;
-	}
-
-	if (bpf_map_push_elem(cgq, &pid, 0)) {
-		scx_bpf_error("cgroup[%llu] queue overflow", cgid);
-		return;
-	}
-
-	/* bump q len, if going 0 -> 1, queue cgroup into the top_q */
-	cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]);
-	if (!cgq_len) {
-		scx_bpf_error("MEMBER_VTPR malfunction");
-		return;
-	}
-
-	if (!__sync_fetch_and_add(cgq_len, 1) &&
-	    bpf_map_push_elem(&top_q, &cgid, 0)) {
-		scx_bpf_error("top_q overflow");
-		return;
-	}
-}
-
-static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask)
-{
-	u32 *vptr;
-
-	vptr = (u32 *)MEMBER_VPTR(pair_id, [cpu]);
-	if (!vptr)
-		return -EINVAL;
-
-	*pairc = bpf_map_lookup_elem(&pair_ctx, vptr);
-	if (!(*pairc))
-		return -EINVAL;
-
-	vptr = (u32 *)MEMBER_VPTR(in_pair_idx, [cpu]);
-	if (!vptr)
-		return -EINVAL;
-
-	*mask = 1U << *vptr;
-
-	return 0;
-}
-
-static int try_dispatch(s32 cpu)
-{
-	struct pair_ctx *pairc;
-	struct bpf_map *cgq_map;
-	struct task_struct *p;
-	u64 now = bpf_ktime_get_ns();
-	bool kick_pair = false;
-	bool expired, pair_preempted;
-	u32 *vptr, in_pair_mask;
-	s32 pid, q_idx;
-	u64 cgid;
-	int ret;
-
-	ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
-	if (ret) {
-		scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]",
-			      cpu);
-		return -ENOENT;
-	}
-
-	bpf_spin_lock(&pairc->lock);
-	pairc->active_mask &= ~in_pair_mask;
-
-	expired = time_before(pairc->started_at + pair_batch_dur_ns, now);
-	if (expired || pairc->draining) {
-		u64 new_cgid = 0;
-
-		__sync_fetch_and_add(&nr_exps, 1);
-
-		/*
-		 * We're done with the current cgid. An obvious optimization
-		 * would be not draining if the next cgroup is the current one.
-		 * For now, be dumb and always expire.
-		 */
-		pairc->draining = true;
-
-		pair_preempted = pairc->preempted_mask;
-		if (pairc->active_mask || pair_preempted) {
-			/*
-			 * The other CPU is still active, or is no longer under
-			 * our control due to e.g. being preempted by a higher
-			 * priority sched_class. We want to wait until this
-			 * cgroup expires, or until control of our pair CPU has
-			 * been returned to us.
-			 *
-			 * If the pair controls its CPU, and the time already
-			 * expired, kick.  When the other CPU arrives at
-			 * dispatch and clears its active mask, it'll push the
-			 * pair to the next cgroup and kick this CPU.
-			 */
-			__sync_fetch_and_add(&nr_exp_waits, 1);
-			bpf_spin_unlock(&pairc->lock);
-			if (expired && !pair_preempted)
-				kick_pair = true;
-			goto out_maybe_kick;
-		}
-
-		bpf_spin_unlock(&pairc->lock);
-
-		/*
-		 * Pick the next cgroup. It'd be easier / cleaner to not drop
-		 * pairc->lock and use stronger synchronization here especially
-		 * given that we'll be switching cgroups significantly less
-		 * frequently than tasks. Unfortunately, bpf_spin_lock can't
-		 * really protect anything non-trivial. Let's do opportunistic
-		 * operations instead.
-		 */
-		bpf_repeat(BPF_MAX_LOOPS) {
-			u32 *q_idx;
-			u64 *cgq_len;
-
-			if (bpf_map_pop_elem(&top_q, &new_cgid)) {
-				/* no active cgroup, go idle */
-				__sync_fetch_and_add(&nr_exp_empty, 1);
-				return 0;
-			}
-
-			q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid);
-			if (!q_idx)
-				continue;
-
-			/*
-			 * This is the only place where empty cgroups are taken
-			 * off the top_q.
-			 */
-			cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]);
-			if (!cgq_len || !*cgq_len)
-				continue;
-
-			/*
-			 * If it has any tasks, requeue as we may race and not
-			 * execute it.
-			 */
-			bpf_map_push_elem(&top_q, &new_cgid, 0);
-			break;
-		}
-
-		bpf_spin_lock(&pairc->lock);
-
-		/*
-		 * The other CPU may already have started on a new cgroup while
-		 * we dropped the lock. Make sure that we're still draining and
-		 * start on the new cgroup.
-		 */
-		if (pairc->draining && !pairc->active_mask) {
-			__sync_fetch_and_add(&nr_cgrp_next, 1);
-			pairc->cgid = new_cgid;
-			pairc->started_at = now;
-			pairc->draining = false;
-			kick_pair = true;
-		} else {
-			__sync_fetch_and_add(&nr_cgrp_coll, 1);
-		}
-	}
-
-	cgid = pairc->cgid;
-	pairc->active_mask |= in_pair_mask;
-	bpf_spin_unlock(&pairc->lock);
-
-	/* again, it'd be better to do all these with the lock held, oh well */
-	vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid);
-	if (!vptr) {
-		scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid);
-		return -ENOENT;
-	}
-	q_idx = *vptr;
-
-	/* claim one task from cgrp_q w/ q_idx */
-	bpf_repeat(BPF_MAX_LOOPS) {
-		u64 *cgq_len, len;
-
-		cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]);
-		if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) {
-			/* the cgroup must be empty, expire and repeat */
-			__sync_fetch_and_add(&nr_cgrp_empty, 1);
-			bpf_spin_lock(&pairc->lock);
-			pairc->draining = true;
-			pairc->active_mask &= ~in_pair_mask;
-			bpf_spin_unlock(&pairc->lock);
-			return -EAGAIN;
-		}
-
-		if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len)
-			continue;
-
-		break;
-	}
-
-	cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx);
-	if (!cgq_map) {
-		scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]",
-			      cgid, q_idx);
-		return -ENOENT;
-	}
-
-	if (bpf_map_pop_elem(cgq_map, &pid)) {
-		scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]",
-			      cgid, q_idx);
-		return -ENOENT;
-	}
-
-	p = bpf_task_from_pid(pid);
-	if (p) {
-		__sync_fetch_and_add(&nr_dispatched, 1);
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
-		bpf_task_release(p);
-	} else {
-		/* we don't handle dequeues, retry on lost tasks */
-		__sync_fetch_and_add(&nr_missing, 1);
-		return -EAGAIN;
-	}
-
-out_maybe_kick:
-	if (kick_pair) {
-		s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]);
-		if (pair) {
-			__sync_fetch_and_add(&nr_kicks, 1);
-			scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT);
-		}
-	}
-	return 0;
-}
-
-void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev)
-{
-	bpf_repeat(BPF_MAX_LOOPS) {
-		if (try_dispatch(cpu) != -EAGAIN)
-			break;
-	}
-}
-
-void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args)
-{
-	int ret;
-	u32 in_pair_mask;
-	struct pair_ctx *pairc;
-	bool kick_pair;
-
-	ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
-	if (ret)
-		return;
-
-	bpf_spin_lock(&pairc->lock);
-	pairc->preempted_mask &= ~in_pair_mask;
-	/* Kick the pair CPU, unless it was also preempted. */
-	kick_pair = !pairc->preempted_mask;
-	bpf_spin_unlock(&pairc->lock);
-
-	if (kick_pair) {
-		s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]);
-
-		if (pair) {
-			__sync_fetch_and_add(&nr_kicks, 1);
-			scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT);
-		}
-	}
-}
-
-void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args)
-{
-	int ret;
-	u32 in_pair_mask;
-	struct pair_ctx *pairc;
-	bool kick_pair;
-
-	ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
-	if (ret)
-		return;
-
-	bpf_spin_lock(&pairc->lock);
-	pairc->preempted_mask |= in_pair_mask;
-	pairc->active_mask &= ~in_pair_mask;
-	/* Kick the pair CPU if it's still running. */
-	kick_pair = pairc->active_mask;
-	pairc->draining = true;
-	bpf_spin_unlock(&pairc->lock);
-
-	if (kick_pair) {
-		s32 *pair = (s32 *)MEMBER_VPTR(pair_cpu, [cpu]);
-
-		if (pair) {
-			__sync_fetch_and_add(&nr_kicks, 1);
-			scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT);
-		}
-	}
-	__sync_fetch_and_add(&nr_preemptions, 1);
-}
-
-s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp)
-{
-	u64 cgid = cgrp->kn->id;
-	s32 i, q_idx;
-
-	bpf_for(i, 0, MAX_CGRPS) {
-		q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS;
-		if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1))
-			break;
-	}
-	if (i == MAX_CGRPS)
-		return -EBUSY;
-
-	if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) {
-		u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]);
-		if (busy)
-			*busy = 0;
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
-void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp)
-{
-	u64 cgid = cgrp->kn->id;
-	s32 *q_idx;
-
-	q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid);
-	if (q_idx) {
-		u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]);
-		if (busy)
-			*busy = 0;
-		bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid);
-	}
-}
-
-s32 BPF_STRUCT_OPS(pair_init)
-{
-	if (!switch_partial)
-		scx_bpf_switch_all();
-	return 0;
-}
-
-void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei)
-{
-	uei_record(&uei, ei);
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops pair_ops = {
-	.enqueue		= (void *)pair_enqueue,
-	.dispatch		= (void *)pair_dispatch,
-	.cpu_acquire		= (void *)pair_cpu_acquire,
-	.cpu_release		= (void *)pair_cpu_release,
-	.cgroup_init		= (void *)pair_cgroup_init,
-	.cgroup_exit		= (void *)pair_cgroup_exit,
-	.init			= (void *)pair_init,
-	.exit			= (void *)pair_exit,
-	.name			= "pair",
-};
diff --git a/tools/sched_ext/scx_example_pair.c b/tools/sched_ext/scx_example_pair.c
deleted file mode 100644
index 18e032bbc173..000000000000
--- a/tools/sched_ext/scx_example_pair.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <unistd.h>
-#include <signal.h>
-#include <assert.h>
-#include <libgen.h>
-#include <bpf/bpf.h>
-#include "user_exit_info.h"
-#include "scx_example_pair.h"
-#include "scx_example_pair.skel.h"
-
-const char help_fmt[] =
-"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n"
-"execute from the same CPU cgroup.\n"
-"\n"
-"See the top-level comment in .bpf.c for more details.\n"
-"\n"
-"Usage: %s [-S STRIDE] [-p]\n"
-"\n"
-"  -S STRIDE     Override CPU pair stride (default: nr_cpus_ids / 2)\n"
-"  -p            Switch only tasks on SCHED_EXT policy intead of all\n"
-"  -h            Display this help and exit\n";
-
-static volatile int exit_req;
-
-static void sigint_handler(int dummy)
-{
-	exit_req = 1;
-}
-
-int main(int argc, char **argv)
-{
-	struct scx_example_pair *skel;
-	struct bpf_link *link;
-	u64 seq = 0;
-	s32 stride, i, opt, outer_fd;
-
-	signal(SIGINT, sigint_handler);
-	signal(SIGTERM, sigint_handler);
-
-	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
-
-	skel = scx_example_pair__open();
-	assert(skel);
-
-	skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus();
-
-	/* pair up the earlier half to the latter by default, override with -s */
-	stride = skel->rodata->nr_cpu_ids / 2;
-
-	while ((opt = getopt(argc, argv, "S:ph")) != -1) {
-		switch (opt) {
-		case 'S':
-			stride = strtoul(optarg, NULL, 0);
-			break;
-		case 'p':
-			skel->rodata->switch_partial = true;
-			break;
-		default:
-			fprintf(stderr, help_fmt, basename(argv[0]));
-			return opt != 'h';
-		}
-	}
-
-	for (i = 0; i < skel->rodata->nr_cpu_ids; i++) {
-		if (skel->rodata->pair_cpu[i] < 0) {
-			skel->rodata->pair_cpu[i] = i + stride;
-			skel->rodata->pair_cpu[i + stride] = i;
-			skel->rodata->pair_id[i] = i;
-			skel->rodata->pair_id[i + stride] = i;
-			skel->rodata->in_pair_idx[i] = 0;
-			skel->rodata->in_pair_idx[i + stride] = 1;
-		}
-	}
-
-	assert(!scx_example_pair__load(skel));
-
-	/*
-	 * Populate the cgrp_q_arr map which is an array containing per-cgroup
-	 * queues. It'd probably be better to do this from BPF but there are too
-	 * many to initialize statically and there's no way to dynamically
-	 * populate from BPF.
-	 */
-	outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr);
-	assert(outer_fd >= 0);
-
-	printf("Initializing");
-        for (i = 0; i < MAX_CGRPS; i++) {
-		s32 inner_fd;
-
-		if (exit_req)
-			break;
-
-		inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0,
-					  sizeof(u32), MAX_QUEUED, NULL);
-		assert(inner_fd >= 0);
-		assert(!bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY));
-		close(inner_fd);
-
-		if (!(i % 10))
-			printf(".");
-		fflush(stdout);
-        }
-	printf("\n");
-
-	/*
-	 * Fully initialized, attach and run.
-	 */
-	link = bpf_map__attach_struct_ops(skel->maps.pair_ops);
-	assert(link);
-
-	while (!exit_req && !uei_exited(&skel->bss->uei)) {
-		printf("[SEQ %lu]\n", seq++);
-		printf(" total:%10lu dispatch:%10lu   missing:%10lu\n",
-		       skel->bss->nr_total,
-		       skel->bss->nr_dispatched,
-		       skel->bss->nr_missing);
-		printf(" kicks:%10lu preemptions:%7lu\n",
-		       skel->bss->nr_kicks,
-		       skel->bss->nr_preemptions);
-		printf("   exp:%10lu exp_wait:%10lu exp_empty:%10lu\n",
-		       skel->bss->nr_exps,
-		       skel->bss->nr_exp_waits,
-		       skel->bss->nr_exp_empty);
-		printf("cgnext:%10lu   cgcoll:%10lu   cgempty:%10lu\n",
-		       skel->bss->nr_cgrp_next,
-		       skel->bss->nr_cgrp_coll,
-		       skel->bss->nr_cgrp_empty);
-		fflush(stdout);
-		sleep(1);
-	}
-
-	bpf_link__destroy(link);
-	uei_print(&skel->bss->uei);
-	scx_example_pair__destroy(skel);
-	return 0;
-}
diff --git a/tools/sched_ext/scx_example_pair.h b/tools/sched_ext/scx_example_pair.h
deleted file mode 100644
index f60b824272f7..000000000000
--- a/tools/sched_ext/scx_example_pair.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#ifndef __SCX_EXAMPLE_PAIR_H
-#define __SCX_EXAMPLE_PAIR_H
-
-enum {
-	MAX_CPUS		= 4096,
-	MAX_QUEUED		= 4096,
-	MAX_CGRPS		= 4096,
-};
-
-#endif /* __SCX_EXAMPLE_PAIR_H */
diff --git a/tools/sched_ext/scx_example_qmap.bpf.c b/tools/sched_ext/scx_example_qmap.bpf.c
deleted file mode 100644
index 579ab21ae403..000000000000
--- a/tools/sched_ext/scx_example_qmap.bpf.c
+++ /dev/null
@@ -1,401 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A simple five-level FIFO queue scheduler.
- *
- * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets
- * assigned to one depending on its compound weight. Each CPU round robins
- * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from
- * queue0, 2 from queue1, 4 from queue2 and so on.
- *
- * This scheduler demonstrates:
- *
- * - BPF-side queueing using PIDs.
- * - Sleepable per-task storage allocation using ops.prep_enable().
- * - Using ops.cpu_release() to handle a higher priority scheduling class taking
- *   the CPU away.
- * - Core-sched support.
- *
- * This scheduler is primarily for demonstration and testing of sched_ext
- * features and unlikely to be useful for actual workloads.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#include "scx_common.bpf.h"
-#include <linux/sched/prio.h>
-
-char _license[] SEC("license") = "GPL";
-
-const volatile u64 slice_ns = SCX_SLICE_DFL;
-const volatile bool switch_partial;
-const volatile u32 stall_user_nth;
-const volatile u32 stall_kernel_nth;
-const volatile u32 dsp_inf_loop_after;
-const volatile s32 disallow_tgid;
-
-u32 test_error_cnt;
-
-struct user_exit_info uei;
-
-struct qmap {
-	__uint(type, BPF_MAP_TYPE_QUEUE);
-	__uint(max_entries, 4096);
-	__type(value, u32);
-} queue0 SEC(".maps"),
-  queue1 SEC(".maps"),
-  queue2 SEC(".maps"),
-  queue3 SEC(".maps"),
-  queue4 SEC(".maps");
-
-struct {
-	__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
-	__uint(max_entries, 5);
-	__type(key, int);
-	__array(values, struct qmap);
-} queue_arr SEC(".maps") = {
-	.values = {
-		[0] = &queue0,
-		[1] = &queue1,
-		[2] = &queue2,
-		[3] = &queue3,
-		[4] = &queue4,
-	},
-};
-
-/*
- * Per-queue sequence numbers to implement core-sched ordering.
- *
- * Tail seq is assigned to each queued task and incremented. Head seq tracks the
- * sequence number of the latest dispatched task. The distance between the a
- * task's seq and the associated queue's head seq is called the queue distance
- * and used when comparing two tasks for ordering. See qmap_core_sched_before().
- */
-static u64 core_sched_head_seqs[5];
-static u64 core_sched_tail_seqs[5];
-
-/* Per-task scheduling context */
-struct task_ctx {
-	bool	force_local;	/* Dispatch directly to local_dsq */
-	u64	core_sched_seq;
-};
-
-struct {
-	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
-	__uint(map_flags, BPF_F_NO_PREALLOC);
-	__type(key, int);
-	__type(value, struct task_ctx);
-} task_ctx_stor SEC(".maps");
-
-/* Per-cpu dispatch index and remaining count */
-struct {
-	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
-	__uint(max_entries, 2);
-	__type(key, u32);
-	__type(value, u64);
-} dispatch_idx_cnt SEC(".maps");
-
-/* Statistics */
-unsigned long nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued;
-unsigned long nr_core_sched_execed;
-
-s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p,
-		   s32 prev_cpu, u64 wake_flags)
-{
-	struct task_ctx *tctx;
-	s32 cpu;
-
-	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
-	if (!tctx) {
-		scx_bpf_error("task_ctx lookup failed");
-		return -ESRCH;
-	}
-
-	if (p->nr_cpus_allowed == 1 ||
-	    scx_bpf_test_and_clear_cpu_idle(prev_cpu)) {
-		tctx->force_local = true;
-		return prev_cpu;
-	}
-
-	cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr);
-	if (cpu >= 0)
-		return cpu;
-
-	return prev_cpu;
-}
-
-static int weight_to_idx(u32 weight)
-{
-	/* Coarsely map the compound weight to a FIFO. */
-	if (weight <= 25)
-		return 0;
-	else if (weight <= 50)
-		return 1;
-	else if (weight < 200)
-		return 2;
-	else if (weight < 400)
-		return 3;
-	else
-		return 4;
-}
-
-void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
-{
-	static u32 user_cnt, kernel_cnt;
-	struct task_ctx *tctx;
-	u32 pid = p->pid;
-	int idx = weight_to_idx(p->scx.weight);
-	void *ring;
-
-	if (p->flags & PF_KTHREAD) {
-		if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth))
-			return;
-	} else {
-		if (stall_user_nth && !(++user_cnt % stall_user_nth))
-			return;
-	}
-
-	if (test_error_cnt && !--test_error_cnt)
-		scx_bpf_error("test triggering error");
-
-	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
-	if (!tctx) {
-		scx_bpf_error("task_ctx lookup failed");
-		return;
-	}
-
-	/*
-	 * All enqueued tasks must have their core_sched_seq updated for correct
-	 * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in
-	 * qmap_ops.flags.
-	 */
-	tctx->core_sched_seq = core_sched_tail_seqs[idx]++;
-
-	/*
-	 * If qmap_select_cpu() is telling us to or this is the last runnable
-	 * task on the CPU, enqueue locally.
-	 */
-	if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) {
-		tctx->force_local = false;
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags);
-		return;
-	}
-
-	/*
-	 * If the task was re-enqueued due to the CPU being preempted by a
-	 * higher priority scheduling class, just re-enqueue the task directly
-	 * on the global DSQ. As we want another CPU to pick it up, find and
-	 * kick an idle CPU.
-	 */
-	if (enq_flags & SCX_ENQ_REENQ) {
-		s32 cpu;
-
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, 0, enq_flags);
-		cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr);
-		if (cpu >= 0)
-			scx_bpf_kick_cpu(cpu, 0);
-		return;
-	}
-
-	ring = bpf_map_lookup_elem(&queue_arr, &idx);
-	if (!ring) {
-		scx_bpf_error("failed to find ring %d", idx);
-		return;
-	}
-
-	/* Queue on the selected FIFO. If the FIFO overflows, punt to global. */
-	if (bpf_map_push_elem(ring, &pid, 0)) {
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, enq_flags);
-		return;
-	}
-
-	__sync_fetch_and_add(&nr_enqueued, 1);
-}
-
-/*
- * The BPF queue map doesn't support removal and sched_ext can handle spurious
- * dispatches. qmap_dequeue() is only used to collect statistics.
- */
-void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags)
-{
-	__sync_fetch_and_add(&nr_dequeued, 1);
-	if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC)
-		__sync_fetch_and_add(&nr_core_sched_execed, 1);
-}
-
-static void update_core_sched_head_seq(struct task_struct *p)
-{
-	struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
-	int idx = weight_to_idx(p->scx.weight);
-
-	if (tctx)
-		core_sched_head_seqs[idx] = tctx->core_sched_seq;
-	else
-		scx_bpf_error("task_ctx lookup failed");
-}
-
-void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev)
-{
-	u32 zero = 0, one = 1;
-	u64 *idx = bpf_map_lookup_elem(&dispatch_idx_cnt, &zero);
-	u64 *cnt = bpf_map_lookup_elem(&dispatch_idx_cnt, &one);
-	void *fifo;
-	s32 pid;
-	int i;
-
-	if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) {
-		struct task_struct *p;
-
-		/*
-		 * PID 2 should be kthreadd which should mostly be idle and off
-		 * the scheduler. Let's keep dispatching it to force the kernel
-		 * to call this function over and over again.
-		 */
-		p = bpf_task_from_pid(2);
-		if (p) {
-			scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0);
-			bpf_task_release(p);
-			return;
-		}
-	}
-
-	if (!idx || !cnt) {
-		scx_bpf_error("failed to lookup idx[%p], cnt[%p]", idx, cnt);
-		return;
-	}
-
-	for (i = 0; i < 5; i++) {
-		/* Advance the dispatch cursor and pick the fifo. */
-		if (!*cnt) {
-			*idx = (*idx + 1) % 5;
-			*cnt = 1 << *idx;
-		}
-		(*cnt)--;
-
-		fifo = bpf_map_lookup_elem(&queue_arr, idx);
-		if (!fifo) {
-			scx_bpf_error("failed to find ring %llu", *idx);
-			return;
-		}
-
-		/* Dispatch or advance. */
-		if (!bpf_map_pop_elem(fifo, &pid)) {
-			struct task_struct *p;
-
-			p = bpf_task_from_pid(pid);
-			if (p) {
-				update_core_sched_head_seq(p);
-				__sync_fetch_and_add(&nr_dispatched, 1);
-				scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, slice_ns, 0);
-				bpf_task_release(p);
-				return;
-			}
-		}
-
-		*cnt = 0;
-	}
-}
-
-/*
- * The distance from the head of the queue scaled by the weight of the queue.
- * The lower the number, the older the task and the higher the priority.
- */
-static s64 task_qdist(struct task_struct *p)
-{
-	int idx = weight_to_idx(p->scx.weight);
-	struct task_ctx *tctx;
-	s64 qdist;
-
-	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
-	if (!tctx) {
-		scx_bpf_error("task_ctx lookup failed");
-		return 0;
-	}
-
-	qdist = tctx->core_sched_seq - core_sched_head_seqs[idx];
-
-	/*
-	 * As queue index increments, the priority doubles. The queue w/ index 3
-	 * is dispatched twice more frequently than 2. Reflect the difference by
-	 * scaling qdists accordingly. Note that the shift amount needs to be
-	 * flipped depending on the sign to avoid flipping priority direction.
-	 */
-	if (qdist >= 0)
-		return qdist << (4 - idx);
-	else
-		return qdist << idx;
-}
-
-/*
- * This is called to determine the task ordering when core-sched is picking
- * tasks to execute on SMT siblings and should encode about the same ordering as
- * the regular scheduling path. Use the priority-scaled distances from the head
- * of the queues to compare the two tasks which should be consistent with the
- * dispatch path behavior.
- */
-bool BPF_STRUCT_OPS(qmap_core_sched_before,
-		    struct task_struct *a, struct task_struct *b)
-{
-	return task_qdist(a) > task_qdist(b);
-}
-
-void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args)
-{
-	u32 cnt;
-
-	/*
-	 * Called when @cpu is taken by a higher priority scheduling class. This
-	 * makes @cpu no longer available for executing sched_ext tasks. As we
-	 * don't want the tasks in @cpu's local dsq to sit there until @cpu
-	 * becomes available again, re-enqueue them into the global dsq. See
-	 * %SCX_ENQ_REENQ handling in qmap_enqueue().
-	 */
-	cnt = scx_bpf_reenqueue_local();
-	if (cnt)
-		__sync_fetch_and_add(&nr_reenqueued, cnt);
-}
-
-s32 BPF_STRUCT_OPS(qmap_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
-{
-	if (p->tgid == disallow_tgid)
-		p->scx.disallow = true;
-
-	/*
-	 * @p is new. Let's ensure that its task_ctx is available. We can sleep
-	 * in this function and the following will automatically use GFP_KERNEL.
-	 */
-	if (bpf_task_storage_get(&task_ctx_stor, p, 0,
-				 BPF_LOCAL_STORAGE_GET_F_CREATE))
-		return 0;
-	else
-		return -ENOMEM;
-}
-
-s32 BPF_STRUCT_OPS(qmap_init)
-{
-	if (!switch_partial)
-		scx_bpf_switch_all();
-	return 0;
-}
-
-void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei)
-{
-	uei_record(&uei, ei);
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops qmap_ops = {
-	.select_cpu		= (void *)qmap_select_cpu,
-	.enqueue		= (void *)qmap_enqueue,
-	.dequeue		= (void *)qmap_dequeue,
-	.dispatch		= (void *)qmap_dispatch,
-	.core_sched_before	= (void *)qmap_core_sched_before,
-	.cpu_release		= (void *)qmap_cpu_release,
-	.prep_enable		= (void *)qmap_prep_enable,
-	.init			= (void *)qmap_init,
-	.exit			= (void *)qmap_exit,
-	.flags			= SCX_OPS_ENQ_LAST,
-	.timeout_ms		= 5000U,
-	.name			= "qmap",
-};
diff --git a/tools/sched_ext/scx_example_qmap.c b/tools/sched_ext/scx_example_qmap.c
deleted file mode 100644
index ccb4814ee61b..000000000000
--- a/tools/sched_ext/scx_example_qmap.c
+++ /dev/null
@@ -1,107 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <signal.h>
-#include <assert.h>
-#include <libgen.h>
-#include <bpf/bpf.h>
-#include "user_exit_info.h"
-#include "scx_example_qmap.skel.h"
-
-const char help_fmt[] =
-"A simple five-level FIFO queue sched_ext scheduler.\n"
-"\n"
-"See the top-level comment in .bpf.c for more details.\n"
-"\n"
-"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-d PID] [-p]\n"
-"\n"
-"  -s SLICE_US   Override slice duration\n"
-"  -e COUNT      Trigger scx_bpf_error() after COUNT enqueues\n"
-"  -t COUNT      Stall every COUNT'th user thread\n"
-"  -T COUNT      Stall every COUNT'th kernel thread\n"
-"  -l COUNT      Trigger dispatch infinite looping after COUNT dispatches\n"
-"  -d PID        Disallow a process from switching into SCHED_EXT (-1 for self)\n"
-"  -p            Switch only tasks on SCHED_EXT policy intead of all\n"
-"  -h            Display this help and exit\n";
-
-static volatile int exit_req;
-
-static void sigint_handler(int dummy)
-{
-	exit_req = 1;
-}
-
-int main(int argc, char **argv)
-{
-	struct scx_example_qmap *skel;
-	struct bpf_link *link;
-	int opt;
-
-	signal(SIGINT, sigint_handler);
-	signal(SIGTERM, sigint_handler);
-
-	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
-
-	skel = scx_example_qmap__open();
-	assert(skel);
-
-	while ((opt = getopt(argc, argv, "s:e:t:T:l:d:ph")) != -1) {
-		switch (opt) {
-		case 's':
-			skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
-			break;
-		case 'e':
-			skel->bss->test_error_cnt = strtoul(optarg, NULL, 0);
-			break;
-		case 't':
-			skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0);
-			break;
-		case 'T':
-			skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0);
-			break;
-		case 'l':
-			skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0);
-			break;
-		case 'd':
-			skel->rodata->disallow_tgid = strtol(optarg, NULL, 0);
-			if (skel->rodata->disallow_tgid < 0)
-				skel->rodata->disallow_tgid = getpid();
-			break;
-		case 'p':
-			skel->rodata->switch_partial = true;
-			break;
-		default:
-			fprintf(stderr, help_fmt, basename(argv[0]));
-			return opt != 'h';
-		}
-	}
-
-	assert(!scx_example_qmap__load(skel));
-
-	link = bpf_map__attach_struct_ops(skel->maps.qmap_ops);
-	assert(link);
-
-	while (!exit_req && !uei_exited(&skel->bss->uei)) {
-		long nr_enqueued = skel->bss->nr_enqueued;
-		long nr_dispatched = skel->bss->nr_dispatched;
-
-		printf("enq=%lu, dsp=%lu, delta=%ld, reenq=%lu, deq=%lu, core=%lu\n",
-		       nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched,
-		       skel->bss->nr_reenqueued, skel->bss->nr_dequeued,
-		       skel->bss->nr_core_sched_execed);
-		fflush(stdout);
-		sleep(1);
-	}
-
-	bpf_link__destroy(link);
-	uei_print(&skel->bss->uei);
-	scx_example_qmap__destroy(skel);
-	return 0;
-}
diff --git a/tools/sched_ext/scx_example_simple.bpf.c b/tools/sched_ext/scx_example_simple.bpf.c
deleted file mode 100644
index 4bccca3e2047..000000000000
--- a/tools/sched_ext/scx_example_simple.bpf.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A simple scheduler.
- *
- * By default, it operates as a simple global weighted vtime scheduler and can
- * be switched to FIFO scheduling. It also demonstrates the following niceties.
- *
- * - Statistics tracking how many tasks are queued to local and global dsq's.
- * - Termination notification for userspace.
- *
- * While very simple, this scheduler should work reasonably well on CPUs with a
- * uniform L3 cache topology. While preemption is not implemented, the fact that
- * the scheduling queue is shared across all CPUs means that whatever is at the
- * front of the queue is likely to be executed fairly quickly given enough
- * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads
- * but comes with the usual problems with FIFO scheduling where saturating
- * threads can easily drown out interactive ones.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#include "scx_common.bpf.h"
-
-char _license[] SEC("license") = "GPL";
-
-const volatile bool fifo_sched;
-const volatile bool switch_partial;
-
-static u64 vtime_now;
-struct user_exit_info uei;
-
-struct {
-	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
-	__uint(key_size, sizeof(u32));
-	__uint(value_size, sizeof(u64));
-	__uint(max_entries, 2);			/* [local, global] */
-} stats SEC(".maps");
-
-static void stat_inc(u32 idx)
-{
-	u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx);
-	if (cnt_p)
-		(*cnt_p)++;
-}
-
-static inline bool vtime_before(u64 a, u64 b)
-{
-	return (s64)(a - b) < 0;
-}
-
-void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
-{
-	/*
-	 * If scx_select_cpu_dfl() is setting %SCX_ENQ_LOCAL, it indicates that
-	 * running @p on its CPU directly shouldn't affect fairness. Just queue
-	 * it on the local FIFO.
-	 */
-	if (enq_flags & SCX_ENQ_LOCAL) {
-		stat_inc(0);	/* count local queueing */
-		scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, enq_flags);
-		return;
-	}
-
-	stat_inc(1);	/* count global queueing */
-
-	if (fifo_sched) {
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
-	} else {
-		u64 vtime = p->scx.dsq_vtime;
-
-		/*
-		 * Limit the amount of budget that an idling task can accumulate
-		 * to one slice.
-		 */
-		if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL))
-			vtime = vtime_now - SCX_SLICE_DFL;
-
-		scx_bpf_dispatch_vtime(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, vtime,
-				       enq_flags);
-	}
-}
-
-void BPF_STRUCT_OPS(simple_running, struct task_struct *p)
-{
-	if (fifo_sched)
-		return;
-
-	/*
-	 * Global vtime always progresses forward as tasks start executing. The
-	 * test and update can be performed concurrently from multiple CPUs and
-	 * thus racy. Any error should be contained and temporary. Let's just
-	 * live with it.
-	 */
-	if (vtime_before(vtime_now, p->scx.dsq_vtime))
-		vtime_now = p->scx.dsq_vtime;
-}
-
-void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable)
-{
-	if (fifo_sched)
-		return;
-
-	/* scale the execution time by the inverse of the weight and charge */
-	p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight;
-}
-
-s32 BPF_STRUCT_OPS(simple_init)
-{
-	if (!switch_partial)
-		scx_bpf_switch_all();
-	return 0;
-}
-
-void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei)
-{
-	uei_record(&uei, ei);
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops simple_ops = {
-	.enqueue		= (void *)simple_enqueue,
-	.running		= (void *)simple_running,
-	.stopping		= (void *)simple_stopping,
-	.init			= (void *)simple_init,
-	.exit			= (void *)simple_exit,
-	.name			= "simple",
-};
diff --git a/tools/sched_ext/scx_example_simple.c b/tools/sched_ext/scx_example_simple.c
deleted file mode 100644
index 486b401f7c95..000000000000
--- a/tools/sched_ext/scx_example_simple.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <unistd.h>
-#include <signal.h>
-#include <assert.h>
-#include <libgen.h>
-#include <bpf/bpf.h>
-#include "user_exit_info.h"
-#include "scx_example_simple.skel.h"
-
-const char help_fmt[] =
-"A simple sched_ext scheduler.\n"
-"\n"
-"See the top-level comment in .bpf.c for more details.\n"
-"\n"
-"Usage: %s [-f] [-p]\n"
-"\n"
-"  -f            Use FIFO scheduling instead of weighted vtime scheduling\n"
-"  -p            Switch only tasks on SCHED_EXT policy intead of all\n"
-"  -h            Display this help and exit\n";
-
-static volatile int exit_req;
-
-static void sigint_handler(int simple)
-{
-	exit_req = 1;
-}
-
-static void read_stats(struct scx_example_simple *skel, u64 *stats)
-{
-	int nr_cpus = libbpf_num_possible_cpus();
-	u64 cnts[2][nr_cpus];
-	u32 idx;
-
-	memset(stats, 0, sizeof(stats[0]) * 2);
-
-	for (idx = 0; idx < 2; idx++) {
-		int ret, cpu;
-
-		ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats),
-					  &idx, cnts[idx]);
-		if (ret < 0)
-			continue;
-		for (cpu = 0; cpu < nr_cpus; cpu++)
-			stats[idx] += cnts[idx][cpu];
-	}
-}
-
-int main(int argc, char **argv)
-{
-	struct scx_example_simple *skel;
-	struct bpf_link *link;
-	u32 opt;
-
-	signal(SIGINT, sigint_handler);
-	signal(SIGTERM, sigint_handler);
-
-	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
-
-	skel = scx_example_simple__open();
-	assert(skel);
-
-	while ((opt = getopt(argc, argv, "fph")) != -1) {
-		switch (opt) {
-		case 'f':
-			skel->rodata->fifo_sched = true;
-			break;
-		case 'p':
-			skel->rodata->switch_partial = true;
-			break;
-		default:
-			fprintf(stderr, help_fmt, basename(argv[0]));
-			return opt != 'h';
-		}
-	}
-
-	assert(!scx_example_simple__load(skel));
-
-	link = bpf_map__attach_struct_ops(skel->maps.simple_ops);
-	assert(link);
-
-	while (!exit_req && !uei_exited(&skel->bss->uei)) {
-		u64 stats[2];
-
-		read_stats(skel, stats);
-		printf("local=%lu global=%lu\n", stats[0], stats[1]);
-		fflush(stdout);
-		sleep(1);
-	}
-
-	bpf_link__destroy(link);
-	uei_print(&skel->bss->uei);
-	scx_example_simple__destroy(skel);
-	return 0;
-}
diff --git a/tools/sched_ext/scx_example_userland.bpf.c b/tools/sched_ext/scx_example_userland.bpf.c
deleted file mode 100644
index a089bc6bbe86..000000000000
--- a/tools/sched_ext/scx_example_userland.bpf.c
+++ /dev/null
@@ -1,269 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A minimal userland scheduler.
- *
- * In terms of scheduling, this provides two different types of behaviors:
- * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity.
- *    All such tasks are direct-dispatched from the kernel, and are never
- *    enqueued in user space.
- * 2. A primitive vruntime scheduler that is implemented in user space, for all
- *    other tasks.
- *
- * Some parts of this example user space scheduler could be implemented more
- * efficiently using more complex and sophisticated data structures. For
- * example, rather than using BPF_MAP_TYPE_QUEUE's,
- * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between
- * user space and kernel space. Similarly, we use a simple vruntime-sorted list
- * in user space, but an rbtree could be used instead.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#include <string.h>
-#include "scx_common.bpf.h"
-#include "scx_example_userland_common.h"
-
-char _license[] SEC("license") = "GPL";
-
-const volatile bool switch_partial;
-const volatile s32 usersched_pid;
-
-/* !0 for veristat, set during init */
-const volatile u32 num_possible_cpus = 64;
-
-/* Stats that are printed by user space. */
-u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues;
-
-struct user_exit_info uei;
-
-/*
- * Whether the user space scheduler needs to be scheduled due to a task being
- * enqueued in user space.
- */
-static bool usersched_needed;
-
-/*
- * The map containing tasks that are enqueued in user space from the kernel.
- *
- * This map is drained by the user space scheduler.
- */
-struct {
-	__uint(type, BPF_MAP_TYPE_QUEUE);
-	__uint(max_entries, USERLAND_MAX_TASKS);
-	__type(value, struct scx_userland_enqueued_task);
-} enqueued SEC(".maps");
-
-/*
- * The map containing tasks that are dispatched to the kernel from user space.
- *
- * Drained by the kernel in userland_dispatch().
- */
-struct {
-	__uint(type, BPF_MAP_TYPE_QUEUE);
-	__uint(max_entries, USERLAND_MAX_TASKS);
-	__type(value, s32);
-} dispatched SEC(".maps");
-
-/* Per-task scheduling context */
-struct task_ctx {
-	bool force_local; /* Dispatch directly to local DSQ */
-};
-
-/* Map that contains task-local storage. */
-struct {
-	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
-	__uint(map_flags, BPF_F_NO_PREALLOC);
-	__type(key, int);
-	__type(value, struct task_ctx);
-} task_ctx_stor SEC(".maps");
-
-static bool is_usersched_task(const struct task_struct *p)
-{
-	return p->pid == usersched_pid;
-}
-
-static bool keep_in_kernel(const struct task_struct *p)
-{
-	return p->nr_cpus_allowed < num_possible_cpus;
-}
-
-static struct task_struct *usersched_task(void)
-{
-	struct task_struct *p;
-
-	p = bpf_task_from_pid(usersched_pid);
-	/*
-	 * Should never happen -- the usersched task should always be managed
-	 * by sched_ext.
-	 */
-	if (!p) {
-		scx_bpf_error("Failed to find usersched task %d", usersched_pid);
-		/*
-		 * We should never hit this path, and we error out of the
-		 * scheduler above just in case, so the scheduler will soon be
-		 * be evicted regardless. So as to simplify the logic in the
-		 * caller to not have to check for NULL, return an acquired
-		 * reference to the current task here rather than NULL.
-		 */
-		return bpf_task_acquire(bpf_get_current_task_btf());
-	}
-
-	return p;
-}
-
-s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p,
-		   s32 prev_cpu, u64 wake_flags)
-{
-	if (keep_in_kernel(p)) {
-		s32 cpu;
-		struct task_ctx *tctx;
-
-		tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
-		if (!tctx) {
-			scx_bpf_error("Failed to look up task-local storage for %s", p->comm);
-			return -ESRCH;
-		}
-
-		if (p->nr_cpus_allowed == 1 ||
-		    scx_bpf_test_and_clear_cpu_idle(prev_cpu)) {
-			tctx->force_local = true;
-			return prev_cpu;
-		}
-
-		cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr);
-		if (cpu >= 0) {
-			tctx->force_local = true;
-			return cpu;
-		}
-	}
-
-	return prev_cpu;
-}
-
-static void dispatch_user_scheduler(void)
-{
-	struct task_struct *p;
-
-	usersched_needed = false;
-	p = usersched_task();
-	scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
-	bpf_task_release(p);
-}
-
-static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags)
-{
-	struct scx_userland_enqueued_task task;
-
-	memset(&task, 0, sizeof(task));
-	task.pid = p->pid;
-	task.sum_exec_runtime = p->se.sum_exec_runtime;
-	task.weight = p->scx.weight;
-
-	if (bpf_map_push_elem(&enqueued, &task, 0)) {
-		/*
-		 * If we fail to enqueue the task in user space, put it
-		 * directly on the global DSQ.
-		 */
-		__sync_fetch_and_add(&nr_failed_enqueues, 1);
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
-	} else {
-		__sync_fetch_and_add(&nr_user_enqueues, 1);
-		usersched_needed = true;
-	}
-}
-
-void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags)
-{
-	if (keep_in_kernel(p)) {
-		u64 dsq_id = SCX_DSQ_GLOBAL;
-		struct task_ctx *tctx;
-
-		tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
-		if (!tctx) {
-			scx_bpf_error("Failed to lookup task ctx for %s", p->comm);
-			return;
-		}
-
-		if (tctx->force_local)
-			dsq_id = SCX_DSQ_LOCAL;
-		tctx->force_local = false;
-		scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags);
-		__sync_fetch_and_add(&nr_kernel_enqueues, 1);
-		return;
-	} else if (!is_usersched_task(p)) {
-		enqueue_task_in_user_space(p, enq_flags);
-	}
-}
-
-void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev)
-{
-	if (usersched_needed)
-		dispatch_user_scheduler();
-
-	bpf_repeat(4096) {
-		s32 pid;
-		struct task_struct *p;
-
-		if (bpf_map_pop_elem(&dispatched, &pid))
-			break;
-
-		/*
-		 * The task could have exited by the time we get around to
-		 * dispatching it. Treat this as a normal occurrence, and simply
-		 * move onto the next iteration.
-		 */
-		p = bpf_task_from_pid(pid);
-		if (!p)
-			continue;
-
-		scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
-		bpf_task_release(p);
-	}
-}
-
-s32 BPF_STRUCT_OPS(userland_prep_enable, struct task_struct *p,
-		   struct scx_enable_args *args)
-{
-	if (bpf_task_storage_get(&task_ctx_stor, p, 0,
-				 BPF_LOCAL_STORAGE_GET_F_CREATE))
-		return 0;
-	else
-		return -ENOMEM;
-}
-
-s32 BPF_STRUCT_OPS(userland_init)
-{
-	if (num_possible_cpus == 0) {
-		scx_bpf_error("User scheduler # CPUs uninitialized (%d)",
-			      num_possible_cpus);
-		return -EINVAL;
-	}
-
-	if (usersched_pid <= 0) {
-		scx_bpf_error("User scheduler pid uninitialized (%d)",
-			      usersched_pid);
-		return -EINVAL;
-	}
-
-	if (!switch_partial)
-		scx_bpf_switch_all();
-	return 0;
-}
-
-void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei)
-{
-	uei_record(&uei, ei);
-}
-
-SEC(".struct_ops")
-struct sched_ext_ops userland_ops = {
-	.select_cpu		= (void *)userland_select_cpu,
-	.enqueue		= (void *)userland_enqueue,
-	.dispatch		= (void *)userland_dispatch,
-	.prep_enable		= (void *)userland_prep_enable,
-	.init			= (void *)userland_init,
-	.exit			= (void *)userland_exit,
-	.timeout_ms		= 3000,
-	.name			= "userland",
-};
diff --git a/tools/sched_ext/scx_example_userland.c b/tools/sched_ext/scx_example_userland.c
deleted file mode 100644
index 4152b1e65fe1..000000000000
--- a/tools/sched_ext/scx_example_userland.c
+++ /dev/null
@@ -1,402 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * A demo sched_ext user space scheduler which provides vruntime semantics
- * using a simple ordered-list implementation.
- *
- * Each CPU in the system resides in a single, global domain. This precludes
- * the need to do any load balancing between domains. The scheduler could
- * easily be extended to support multiple domains, with load balancing
- * happening in user space.
- *
- * Any task which has any CPU affinity is scheduled entirely in BPF. This
- * program only schedules tasks which may run on any CPU.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <unistd.h>
-#include <sched.h>
-#include <signal.h>
-#include <assert.h>
-#include <libgen.h>
-#include <pthread.h>
-#include <bpf/bpf.h>
-#include <sys/mman.h>
-#include <sys/queue.h>
-#include <sys/syscall.h>
-
-#include "user_exit_info.h"
-#include "scx_example_userland_common.h"
-#include "scx_example_userland.skel.h"
-
-const char help_fmt[] =
-"A minimal userland sched_ext scheduler.\n"
-"\n"
-"See the top-level comment in .bpf.c for more details.\n"
-"\n"
-"Usage: %s [-b BATCH] [-p]\n"
-"\n"
-"  -b BATCH      The number of tasks to batch when dispatching (default: 8)\n"
-"  -p            Don't switch all, switch only tasks on SCHED_EXT policy\n"
-"  -h            Display this help and exit\n";
-
-/* Defined in UAPI */
-#define SCHED_EXT 7
-
-/* Number of tasks to batch when dispatching to user space. */
-static __u32 batch_size = 8;
-
-static volatile int exit_req;
-static int enqueued_fd, dispatched_fd;
-
-static struct scx_example_userland *skel;
-static struct bpf_link *ops_link;
-
-/* Stats collected in user space. */
-static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches;
-
-/* The data structure containing tasks that are enqueued in user space. */
-struct enqueued_task {
-	LIST_ENTRY(enqueued_task) entries;
-	__u64 sum_exec_runtime;
-	double vruntime;
-};
-
-/*
- * Use a vruntime-sorted list to store tasks. This could easily be extended to
- * a more optimal data structure, such as an rbtree as is done in CFS. We
- * currently elect to use a sorted list to simplify the example for
- * illustrative purposes.
- */
-LIST_HEAD(listhead, enqueued_task);
-
-/*
- * A vruntime-sorted list of tasks. The head of the list contains the task with
- * the lowest vruntime. That is, the task that has the "highest" claim to be
- * scheduled.
- */
-static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head);
-
-/*
- * The statically allocated array of tasks. We use a statically allocated list
- * here to avoid having to allocate on the enqueue path, which could cause a
- * deadlock. A more substantive user space scheduler could e.g. provide a hook
- * for newly enabled tasks that are passed to the scheduler from the
- * .prep_enable() callback to allows the scheduler to allocate on safe paths.
- */
-struct enqueued_task tasks[USERLAND_MAX_TASKS];
-
-static double min_vruntime;
-
-static void sigint_handler(int userland)
-{
-	exit_req = 1;
-}
-
-static __u32 task_pid(const struct enqueued_task *task)
-{
-	return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task);
-}
-
-static int dispatch_task(s32 pid)
-{
-	int err;
-
-	err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0);
-	if (err) {
-		fprintf(stderr, "Failed to dispatch task %d\n", pid);
-		exit_req = 1;
-	} else {
-		nr_vruntime_dispatches++;
-	}
-
-	return err;
-}
-
-static struct enqueued_task *get_enqueued_task(__s32 pid)
-{
-	if (pid >= USERLAND_MAX_TASKS)
-		return NULL;
-
-	return &tasks[pid];
-}
-
-static double calc_vruntime_delta(__u64 weight, __u64 delta)
-{
-	double weight_f = (double)weight / 100.0;
-	double delta_f = (double)delta;
-
-	return delta_f / weight_f;
-}
-
-static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task)
-{
-	__u64 delta;
-
-	delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime;
-
-	enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta);
-	if (min_vruntime > enqueued->vruntime)
-		enqueued->vruntime = min_vruntime;
-	enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime;
-}
-
-static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task)
-{
-	struct enqueued_task *curr, *enqueued, *prev;
-
-	curr = get_enqueued_task(bpf_task->pid);
-	if (!curr)
-		return ENOENT;
-
-	update_enqueued(curr, bpf_task);
-	nr_vruntime_enqueues++;
-
-	/*
-	 * Enqueue the task in a vruntime-sorted list. A more optimal data
-	 * structure such as an rbtree could easily be used as well. We elect
-	 * to use a list here simply because it's less code, and thus the
-	 * example is less convoluted and better serves to illustrate what a
-	 * user space scheduler could look like.
-	 */
-
-	if (LIST_EMPTY(&vruntime_head)) {
-		LIST_INSERT_HEAD(&vruntime_head, curr, entries);
-		return 0;
-	}
-
-	LIST_FOREACH(enqueued, &vruntime_head, entries) {
-		if (curr->vruntime <= enqueued->vruntime) {
-			LIST_INSERT_BEFORE(enqueued, curr, entries);
-			return 0;
-		}
-		prev = enqueued;
-	}
-
-	LIST_INSERT_AFTER(prev, curr, entries);
-
-	return 0;
-}
-
-static void drain_enqueued_map(void)
-{
-	while (1) {
-		struct scx_userland_enqueued_task task;
-		int err;
-
-		if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task))
-			return;
-
-		err = vruntime_enqueue(&task);
-		if (err) {
-			fprintf(stderr, "Failed to enqueue task %d: %s\n",
-				task.pid, strerror(err));
-			exit_req = 1;
-			return;
-		}
-	}
-}
-
-static void dispatch_batch(void)
-{
-	__u32 i;
-
-	for (i = 0; i < batch_size; i++) {
-		struct enqueued_task *task;
-		int err;
-		__s32 pid;
-
-		task = LIST_FIRST(&vruntime_head);
-		if (!task)
-			return;
-
-		min_vruntime = task->vruntime;
-		pid = task_pid(task);
-		LIST_REMOVE(task, entries);
-		err = dispatch_task(pid);
-		if (err) {
-			fprintf(stderr, "Failed to dispatch task %d in %u\n",
-				pid, i);
-			return;
-		}
-	}
-}
-
-static void *run_stats_printer(void *arg)
-{
-	while (!exit_req) {
-		__u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total;
-
-		nr_failed_enqueues = skel->bss->nr_failed_enqueues;
-		nr_kernel_enqueues = skel->bss->nr_kernel_enqueues;
-		nr_user_enqueues = skel->bss->nr_user_enqueues;
-		total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues;
-
-		printf("o-----------------------o\n");
-		printf("| BPF ENQUEUES          |\n");
-		printf("|-----------------------|\n");
-		printf("|  kern:     %10llu |\n", nr_kernel_enqueues);
-		printf("|  user:     %10llu |\n", nr_user_enqueues);
-		printf("|  failed:   %10llu |\n", nr_failed_enqueues);
-		printf("|  -------------------- |\n");
-		printf("|  total:    %10llu |\n", total);
-		printf("|                       |\n");
-		printf("|-----------------------|\n");
-		printf("| VRUNTIME / USER       |\n");
-		printf("|-----------------------|\n");
-		printf("|  enq:      %10llu |\n", nr_vruntime_enqueues);
-		printf("|  disp:     %10llu |\n", nr_vruntime_dispatches);
-		printf("o-----------------------o\n");
-		printf("\n\n");
-		sleep(1);
-	}
-
-	return NULL;
-}
-
-static int spawn_stats_thread(void)
-{
-	pthread_t stats_printer;
-
-	return pthread_create(&stats_printer, NULL, run_stats_printer, NULL);
-}
-
-static int bootstrap(int argc, char **argv)
-{
-	int err;
-	__u32 opt;
-	struct sched_param sched_param = {
-		.sched_priority = sched_get_priority_max(SCHED_EXT),
-	};
-	bool switch_partial = false;
-
-	signal(SIGINT, sigint_handler);
-	signal(SIGTERM, sigint_handler);
-	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
-
-	/*
-	 * Enforce that the user scheduler task is managed by sched_ext. The
-	 * task eagerly drains the list of enqueued tasks in its main work
-	 * loop, and then yields the CPU. The BPF scheduler only schedules the
-	 * user space scheduler task when at least one other task in the system
-	 * needs to be scheduled.
-	 */
-	err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param);
-	if (err) {
-		fprintf(stderr, "Failed to set scheduler to SCHED_EXT: %s\n", strerror(err));
-		return err;
-	}
-
-	while ((opt = getopt(argc, argv, "b:ph")) != -1) {
-		switch (opt) {
-		case 'b':
-			batch_size = strtoul(optarg, NULL, 0);
-			break;
-		case 'p':
-			switch_partial = true;
-			break;
-		default:
-			fprintf(stderr, help_fmt, basename(argv[0]));
-			exit(opt != 'h');
-		}
-	}
-
-	/*
-	 * It's not always safe to allocate in a user space scheduler, as an
-	 * enqueued task could hold a lock that we require in order to be able
-	 * to allocate.
-	 */
-	err = mlockall(MCL_CURRENT | MCL_FUTURE);
-	if (err) {
-		fprintf(stderr, "Failed to prefault and lock address space: %s\n",
-			strerror(err));
-		return err;
-	}
-
-	skel = scx_example_userland__open();
-	if (!skel) {
-		fprintf(stderr, "Failed to open scheduler: %s\n", strerror(errno));
-		return errno;
-	}
-	skel->rodata->num_possible_cpus = libbpf_num_possible_cpus();
-	assert(skel->rodata->num_possible_cpus > 0);
-	skel->rodata->usersched_pid = getpid();
-	assert(skel->rodata->usersched_pid > 0);
-	skel->rodata->switch_partial = switch_partial;
-
-	err = scx_example_userland__load(skel);
-	if (err) {
-		fprintf(stderr, "Failed to load scheduler: %s\n", strerror(err));
-		goto destroy_skel;
-	}
-
-	enqueued_fd = bpf_map__fd(skel->maps.enqueued);
-	dispatched_fd = bpf_map__fd(skel->maps.dispatched);
-	assert(enqueued_fd > 0);
-	assert(dispatched_fd > 0);
-
-	err = spawn_stats_thread();
-	if (err) {
-		fprintf(stderr, "Failed to spawn stats thread: %s\n", strerror(err));
-		goto destroy_skel;
-	}
-
-	ops_link = bpf_map__attach_struct_ops(skel->maps.userland_ops);
-	if (!ops_link) {
-		fprintf(stderr, "Failed to attach struct ops: %s\n", strerror(errno));
-		err = errno;
-		goto destroy_skel;
-	}
-
-	return 0;
-
-destroy_skel:
-	scx_example_userland__destroy(skel);
-	exit_req = 1;
-	return err;
-}
-
-static void sched_main_loop(void)
-{
-	while (!exit_req) {
-		/*
-		 * Perform the following work in the main user space scheduler
-		 * loop:
-		 *
-		 * 1. Drain all tasks from the enqueued map, and enqueue them
-		 *    to the vruntime sorted list.
-		 *
-		 * 2. Dispatch a batch of tasks from the vruntime sorted list
-		 *    down to the kernel.
-		 *
-		 * 3. Yield the CPU back to the system. The BPF scheduler will
-		 *    reschedule the user space scheduler once another task has
-		 *    been enqueued to user space.
-		 */
-		drain_enqueued_map();
-		dispatch_batch();
-		sched_yield();
-	}
-}
-
-int main(int argc, char **argv)
-{
-	int err;
-
-	err = bootstrap(argc, argv);
-	if (err) {
-		fprintf(stderr, "Failed to bootstrap scheduler: %s\n", strerror(err));
-		return err;
-	}
-
-	sched_main_loop();
-
-	exit_req = 1;
-	bpf_link__destroy(ops_link);
-	uei_print(&skel->bss->uei);
-	scx_example_userland__destroy(skel);
-	return 0;
-}
diff --git a/tools/sched_ext/scx_example_userland_common.h b/tools/sched_ext/scx_example_userland_common.h
deleted file mode 100644
index 639c6809c5ff..000000000000
--- a/tools/sched_ext/scx_example_userland_common.h
+++ /dev/null
@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright (c) 2022 Meta, Inc */
-
-#ifndef __SCX_USERLAND_COMMON_H
-#define __SCX_USERLAND_COMMON_H
-
-#define USERLAND_MAX_TASKS 8192
-
-/*
- * An instance of a task that has been enqueued by the kernel for consumption
- * by a user space global scheduler thread.
- */
-struct scx_userland_enqueued_task {
-	__s32 pid;
-	u64 sum_exec_runtime;
-	u64 weight;
-};
-
-#endif  // __SCX_USERLAND_COMMON_H
diff --git a/tools/sched_ext/user_exit_info.h b/tools/sched_ext/user_exit_info.h
deleted file mode 100644
index e701ef0e0b86..000000000000
--- a/tools/sched_ext/user_exit_info.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Define struct user_exit_info which is shared between BPF and userspace parts
- * to communicate exit status and other information.
- *
- * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
- * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
- * Copyright (c) 2022 David Vernet <dvernet@meta.com>
- */
-#ifndef __USER_EXIT_INFO_H
-#define __USER_EXIT_INFO_H
-
-struct user_exit_info {
-	int		type;
-	char		reason[128];
-	char		msg[1024];
-};
-
-#ifdef __bpf__
-
-#include "vmlinux.h"
-#include <bpf/bpf_core_read.h>
-
-static inline void uei_record(struct user_exit_info *uei,
-			      const struct scx_exit_info *ei)
-{
-	bpf_probe_read_kernel_str(uei->reason, sizeof(uei->reason), ei->reason);
-	bpf_probe_read_kernel_str(uei->msg, sizeof(uei->msg), ei->msg);
-	/* use __sync to force memory barrier */
-	__sync_val_compare_and_swap(&uei->type, uei->type, ei->type);
-}
-
-#else	/* !__bpf__ */
-
-static inline bool uei_exited(struct user_exit_info *uei)
-{
-	/* use __sync to force memory barrier */
-	return __sync_val_compare_and_swap(&uei->type, -1, -1);
-}
-
-static inline void uei_print(const struct user_exit_info *uei)
-{
-	fprintf(stderr, "EXIT: %s", uei->reason);
-	if (uei->msg[0] != '\0')
-		fprintf(stderr, " (%s)", uei->msg);
-	fputs("\n", stderr);
-}
-
-#endif	/* __bpf__ */
-#endif	/* __USER_EXIT_INFO_H */
diff --git a/kernel/sched/hmbird/hmbird.c b/kernel/sched/hmbird/hmbird.c
index ce05928392bf..f47f79079297 100755
--- a/kernel/sched/hmbird/hmbird.c
+++ b/kernel/sched/hmbird/hmbird.c
@@ -633,14 +633,14 @@ static void init_isolate_cpus(void)
 	WARN_ON(!alloc_cpumask_var(&iso_masks.exclusive, GFP_KERNEL));
 	WARN_ON(!alloc_cpumask_var(&iso_masks.big, GFP_KERNEL));
 	WARN_ON(!alloc_cpumask_var(&iso_masks.little, GFP_KERNEL));
-	cpumask_set_cpu(0, iso_masks.big);
-	cpumask_set_cpu(1, iso_masks.big);
-	cpumask_set_cpu(2, iso_masks.big);
-	cpumask_set_cpu(3, iso_masks.big);
-	cpumask_set_cpu(4, iso_masks.big);
-	cpumask_set_cpu(5, iso_masks.ex_free);
+	cpumask_set_cpu(0, iso_masks.little);
+	cpumask_set_cpu(1, iso_masks.little);
+	cpumask_set_cpu(2, iso_masks.little);
+	cpumask_set_cpu(3, iso_masks.little);
+	cpumask_set_cpu(4, iso_masks.little);
+	cpumask_set_cpu(5, iso_masks.little);
 	cpumask_set_cpu(6, iso_masks.partial);
-	cpumask_set_cpu(7, iso_masks.ex_free);
+	cpumask_set_cpu(7, iso_masks.big);
 }
 
 extern spinlock_t css_set_lock;