starnix_core/task/

kernel.rs

// Copyright 2021 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use crate::bpf::EbpfState;
use crate::device::remote_block_device::RemoteBlockDeviceRegistry;
use crate::device::{DeviceMode, DeviceRegistry};
use crate::execution::CrashReporter;
use crate::mm::{FutexTable, MappingSummary, MlockPinFlavor};
use crate::power::{SuspendResumeManagerHandle, create_watcher_for_wake_events};
use crate::ptrace::StopState;
use crate::security::{self, AuditLogger};
use crate::signals::SignalInfo;
use crate::task::container_namespace::ContainerNamespace;
use crate::task::limits::SystemLimits;
use crate::task::memory_attribution::MemoryAttributionManager;
use crate::task::net::NetstackDevices;
use crate::task::tracing::TracePerformanceEventManager;
use crate::task::{
    AbstractUnixSocketNamespace, AbstractVsockSocketNamespace, CurrentTask, DelayedReleaser,
    ExitStatus, IpTables, KernelCgroups, KernelStats, KernelThreads, PidTable, SchedulerManager,
    Syslog, Task, ThreadGroup, UtsNamespace, UtsNamespaceHandle,
};
use crate::time::{HrTimerManager, HrTimerManagerHandle};
use crate::vdso::vdso_loader::Vdso;
use crate::vfs::fs_args::MountParams;
use crate::vfs::socket::{
    GenericMessage, GenericNetlink, NetlinkAccessControl, NetlinkContextImpl,
    NetlinkToClientSender, SocketAddress, SocketTokensStore,
};
use crate::vfs::{CacheConfig, FileOps, FsNodeHandle, FsString, Mounts, NamespaceNode};
use bstr::{BString, ByteSlice};
use devicetree::types::Devicetree;
use expando::Expando;
use fidl::endpoints::{ClientEnd, DiscoverableProtocolMarker, ProtocolMarker, create_endpoints};
use fidl_fuchsia_component_runner::{ComponentControllerControlHandle, ComponentStopInfo};
use fidl_fuchsia_feedback::CrashReporterProxy;
use fidl_fuchsia_io as fio;
use fidl_fuchsia_memory_attribution as fattribution;
use fidl_fuchsia_net_power as fnet_power;
use fidl_fuchsia_net_resources as fnet_resources;
use fidl_fuchsia_time_external::AdjustSynchronousProxy;
use fuchsia_async as fasync;
use fuchsia_inspect::ArrayProperty;
use fuchsia_rcu::RcuReadScope;
use futures::FutureExt;
use netlink::interfaces::InterfacesHandler;
use netlink::{NETLINK_LOG_TAG, Netlink};
use once_cell::sync::OnceCell;
use scopeguard::ScopeGuard;
use smallvec::SmallVec;
use starnix_lifecycle::AtomicCounter;
use starnix_logging::{SyscallLogFilter, log_debug, log_error, log_info, log_warn};
use starnix_sync::{
    ComponentControllerLock, KernelSwapFiles, LockDepGuard, LockDepMutex, MountsLevel, RwSeqLock,
    RwSeqLockGuard, SyscallLogFiltersLock,
};
use starnix_uapi::device_id::DeviceId;
use starnix_uapi::errors::{Errno, errno};
use starnix_uapi::open_flags::OpenFlags;
use starnix_uapi::signals::SIGKILL;
use starnix_uapi::{VMADDR_CID_HOST, from_status_like_fdio};
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::num::NonZeroU64;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU16, Ordering};
use std::sync::{Arc, OnceLock, Weak};
use zx::{CpuFeatureFlags, CpuSveFeatureFlags};

/// Host kernel free-memory watermarks (in MiB) configured via Product Assembly
/// (`platform.kernel.oom`) and routed to `starnix_kernel` as structured configuration.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct OomWatermarksMb {
    pub out_of_memory_mb: u32,
    pub critical_mb: u32,
    pub warning_mb: u32,
}

/// Kernel features are specified in the component manifest of the starnix container
/// or explicitly provided to the kernel constructor in tests.
#[derive(Debug, Default, Clone)]
pub struct KernelFeatures {
    pub bpf_v2: bool,

    /// Whether the `devtmpfs` filesystem and automatic device node creation are enabled.
    pub devtmpfs: bool,

    /// Whether the kernel supports the S_ISUID and S_ISGID bits.
    ///
    /// For example, these bits are used by `sudo`.
    ///
    /// Enabling this feature is potentially a security risk because they allow privilege
    /// escalation.
    pub enable_suid: bool,

    /// Whether io_uring is enabled.
    ///
    /// TODO(https://fxbug.dev/297431387): Enabled by default once the feature is completed.
    pub io_uring: bool,

    /// Whether the kernel should return an error to userspace, rather than panicking, if `reboot()`
    /// is requested but cannot be enacted because the kernel lacks the relevant capabilities.
    pub error_on_failed_reboot: bool,

    /// The default seclabel that is applied to components that are run in this kernel.
    ///
    /// Components can override this by setting the `seclabel` field in their program block.
    pub default_seclabel: Option<String>,

    /// Whether the kernel is being used to run the SELinux Test Suite.
    ///
    /// TODO: https://fxbug.dev/388077431 - remove this once we no longer need workarounds for the
    /// SELinux Test Suite.
    pub selinux_test_suite: bool,

    /// The default mount options to use when mounting directories from a component's namespace.
    ///
    /// The key is the path in the component's namespace, and the value is the mount options
    /// string.
    pub default_ns_mount_options: Option<HashMap<String, String>>,

    /// The default uid that is applied to components that are run in this kernel.
    ///
    /// Components can override this by setting the `uid` field in their program block.
    pub default_uid: u32,

    /// mlock() never prefaults pages.
    pub mlock_always_onfault: bool,

    /// Implementation of mlock() to use for this kernel instance.
    pub mlock_pin_flavor: MlockPinFlavor,

    /// Whether excessive crash reports should be throttled.
    pub crash_report_throttling: bool,

    /// Whether or not to serve wifi support to Android.
    pub wifi: bool,

    /// The number of bytes to cache in pages for reading zx::MapInfo from VMARs.
    pub cached_zx_map_info_bytes: u32,

    /// The size of the Dirent LRU cache.
    pub dirent_cache_size: u32,

    /// Whether to expose a stub '/dev/ion' node, as a temporary workaround for compatibility.
    // TODO(https://fxbug.dev/485370648) remove when unnecessary
    pub fake_ion: bool,

    /// Whether `remotevol` mounts the volume in-process rather than over FIDL.
    pub remotevol_in_process: bool,

    /// Whether to log unexpected lookup misses in `sysfs` via `track_stub_log!`.
    pub log_sysfs_lookup_misses: bool,

    /// Free-memory watermarks (in MiB) for `/proc/zoneinfo`.
    pub oom_watermarks_mb: Option<OomWatermarksMb>,

    /// Paths of executables whose processes are stopped only after all other processes except init
    /// have exited when the kernel shuts down, e.g. one that writes crash dumps for processes that
    /// crash during shutdown. See `Kernel::run_shutdown()`.
    pub stop_last_on_shutdown: Vec<FsString>,
}

impl KernelFeatures {
    /// Returns the `MountParams` to use when mounting the specified path from a component's
    /// namespace.  This mechanism is also used to specified options for mounts created via
    /// container features, by specifying a pseudo-path e.g. "#container".
    pub fn ns_mount_options(&self, ns_path: &str) -> Result<MountParams, Errno> {
        if let Some(all_options) = &self.default_ns_mount_options {
            if let Some(options) = all_options.get(ns_path) {
                return MountParams::parse(options.as_bytes().into());
            }
        }
        Ok(MountParams::default())
    }
}

/// Kernel command line or module argument structure (`name` or `name=value`).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ArgNameAndValue<'a> {
    pub name: &'a str,
    pub value: Option<&'a str>,
}

impl<'a> From<&'a str> for ArgNameAndValue<'a> {
    fn from(arg: &'a str) -> Self {
        match arg.split_once('=') {
            Some((name, value)) => Self { name, value: Some(value) },
            None => Self { name: arg, value: None },
        }
    }
}

type DeferredDropCallback = Box<dyn FnOnce() + Send + Sync>;

/// A proof token representing the global lock over the namespace mount topology.
///
/// Functions that take `&MountsWriteToken` require the caller to hold the
/// `Kernel::mounts_lock` to ensure safe modification of the global mount tree.
pub struct MountsWriteToken {
    deferred_drops: RefCell<SmallVec<[DeferredDropCallback; 8]>>,
}

impl MountsWriteToken {
    fn new() -> Self {
        Self { deferred_drops: RefCell::new(SmallVec::new()) }
    }

    fn defer_drop_internal<T: Send + Sync + 'static>(&self, value: T) {
        self.deferred_drops.borrow_mut().push(Box::new(move || {
            drop(value);
        }));
    }

    /// We cannot block while holding a [MountsWriteGuard] as it is a spinlock. If a Drop requires
    /// a blocking operation, use [Self::defer_drop] to defer the blocking operation until
    /// after the [MountsWriteGuard] is released.
    pub fn defer_drop<T: Send + Sync + 'static>(
        &self,
        value: T,
    ) -> ScopeGuard<T, impl FnOnce(T) + '_> {
        scopeguard::guard(value, |value| {
            self.defer_drop_internal(value);
        })
    }

    /// Helper method to [Self::defer_drop] specifically for [Arc<T>]s.
    pub fn retain<T: Send + Sync + 'static>(&self, value: &Arc<T>) -> Arc<T> {
        self.defer_drop_internal(value.clone());
        value.clone()
    }

    fn take_deferred_drops(&mut self) -> SmallVec<[DeferredDropCallback; 8]> {
        std::mem::take(self.deferred_drops.get_mut())
    }
}

/// A guard special cased for mount operations to defer destroying mounts until after all
/// modifications of the global mount tree are completed.
pub struct MountsWriteGuard<'a> {
    guard: Option<RwSeqLockGuard<'a, LockDepGuard<'a, MountsWriteToken>>>,
}

impl<'a> MountsWriteGuard<'a> {
    fn new(guard: RwSeqLockGuard<'a, LockDepGuard<'a, MountsWriteToken>>) -> Self {
        Self { guard: Some(guard) }
    }
}

impl<'a> Deref for MountsWriteGuard<'a> {
    type Target = MountsWriteToken;
    fn deref(&self) -> &Self::Target {
        self.guard.as_ref().unwrap()
    }
}

impl<'a> Drop for MountsWriteGuard<'a> {
    fn drop(&mut self) {
        if let Some(mut guard) = self.guard.take() {
            let drops = guard.take_deferred_drops();
            drop(guard);
            for callback in drops {
                callback();
            }
        }
    }
}

/// Maximum time after shutdown starts before deferred-shutdown processes receive SIGKILL.
///
/// Deferred-shutdown processes are processes that are still handling a core-dump signal
/// when their normal shutdown SIGKILL timeout expires (e.g., while writing a crash dump),
/// along with any child processes they spawn. See `ThreadGroup::shut_down()`.
///
/// `component_manager` forcibly kills the container if it does not stop within 5s by
/// default, which skips filesystem unmounts and `on_stop`. Setting this timeout to 3s
/// gives crash dumps time to finish (typically 1-2.5s in testing) while leaving 2s for
/// the remaining shutdown steps (`stop_last_on_shutdown` processes, `init`, unmounts,
/// and `on_stop`, which took up to 0.6s in testing).
// TODO(https://fxbug.dev/574471394): Remove once non-forced shutdown lets the container
// run its own shutdown sequence.
const DEFERRED_SHUTDOWN_TIMEOUT: zx::MonotonicDuration = zx::MonotonicDuration::from_seconds(3);

/// The shared, mutable state for the entire Starnix kernel.
///
/// The `Kernel` object holds all kernel threads, userspace tasks, and file system resources for a
/// single instance of the Starnix kernel. In production, there is one instance of this object for
/// the entire Starnix kernel. However, multiple instances of this object can be created in one
/// process during unit testing.
///
/// The structure of this object will likely need to evolve as we implement more namespacing and
/// isolation mechanisms, such as `namespaces(7)` and `pid_namespaces(7)`.
pub struct Kernel {
    /// Weak reference to self. Allows to not have to pass &Arc<Kernel> in apis.
    pub weak_self: Weak<Kernel>,

    /// The kernel threads running on behalf of this kernel.
    pub kthreads: KernelThreads,

    /// The features enabled for this kernel.
    pub features: KernelFeatures,

    /// The processes and threads running in this kernel, organized by pid_t.
    pub pids: PidTable,

    /// A weak reference to the init task (PID 1).
    pub init_task: OnceLock<Weak<Task>>,

    /// Shared manager of pid/tid to koid mappings for tracing and profiling clients.
    /// A direct field (rather than an expando entry) so the task-creation fast path is a
    /// single relaxed atomic load with no locks when nothing is recording.
    pub trace_event_manager: Arc<TracePerformanceEventManager>,

    /// Subsystem-specific properties that hang off the Kernel object.
    ///
    /// Instead of adding yet another property to the Kernel object, consider storing the property
    /// in an expando if that property is only used by one part of the system, such as a module.
    pub expando: Expando,

    /// The default namespace for abstract AF_UNIX sockets in this kernel.
    ///
    /// Rather than use this default namespace, abstract socket addresses
    /// should be looked up in the AbstractSocketNamespace on each Task
    /// object because some Task objects might have a non-default namespace.
    pub default_abstract_socket_namespace: Arc<AbstractUnixSocketNamespace>,

    /// The default namespace for abstract AF_VSOCK sockets in this kernel.
    pub default_abstract_vsock_namespace: Arc<AbstractVsockSocketNamespace>,

    /// The kernel command line. Shows up in /proc/cmdline.
    pub cmdline: BString,

    pub device_tree: Option<Devicetree>,

    // Global state held by the Linux Security Modules subsystem.
    pub security_state: security::KernelState,

    /// The registry of device drivers.
    pub device_registry: DeviceRegistry,

    /// Mapping of top-level namespace entries to an associated proxy.
    /// For example, "/svc" to the respective proxy. Only the namespace entries
    /// which were known at component startup will be available by the kernel.
    pub container_namespace: ContainerNamespace,

    /// The global lock for the mount tree.
    ///
    /// This lock protects against concurrent modifications to the mount topology. It uses
    /// an `RwSeqLock` to allow readers (like path walking traversing mount points) to get a
    /// consistent, lock-free snapshot of the RCU-protected mount table using `read_seq`.
    /// Writers must acquire the lock before mutating filesystems, moving mounts, or
    /// propagating peer groups. The returned `MountsWriteToken` is used as a proof token
    /// throughout the `namespace` module to statically enforce exclusive write access.
    pub mounts_lock: RwSeqLock<LockDepMutex<MountsWriteToken, MountsLevel>>,

    /// The registry of block devices backed by a remote fuchsia.io file.
    pub remote_block_device_registry: Arc<RemoteBlockDeviceRegistry>,

    /// The iptables used for filtering network packets.
    iptables: OnceLock<IpTables>,

    /// The futexes across all processes in the kernel.
    pub shared_futexes: FutexTable,

    /// The default UTS namespace for all tasks.
    ///
    /// Because each task can have its own UTS namespace, you probably want to use
    /// the UTS namespace handle of the task, which may/may not point to this one.
    pub root_uts_ns: UtsNamespaceHandle,

    /// A struct containing a VMO with a vDSO implementation, if implemented for a given architecture, and possibly an offset for a sigreturn function.
    pub vdso: Vdso,

    /// A struct containing a VMO with a arch32-vDSO implementation, if implemented for a given architecture.
    // TODO(https://fxbug.dev/380431743) This could be made less clunky -- maybe a Vec<Vdso> above or
    // something else
    pub vdso_arch32: Option<Vdso>,

    /// The table of devices installed on the netstack and their associated
    /// state local to this `Kernel`.
    pub netstack_devices: Arc<NetstackDevices>,

    /// Files that are currently available for swapping.
    /// Note: Starnix never actually swaps memory to these files. We just need to track them
    /// to pass conformance tests.
    pub swap_files: LockDepMutex<Vec<FsNodeHandle>, KernelSwapFiles>,

    /// The implementation of generic Netlink protocol families.
    generic_netlink: OnceLock<GenericNetlink<NetlinkToClientSender<GenericMessage>>>,

    /// The implementation of networking-related Netlink protocol families.
    network_netlink: OnceLock<Netlink<NetlinkContextImpl>>,

    /// Inspect instrumentation for this kernel instance.
    pub inspect_node: fuchsia_inspect::Node,

    /// The kinds of seccomp action that gets logged, stored as a bit vector.
    /// Each potential SeccompAction gets a bit in the vector, as specified by
    /// SeccompAction::logged_bit_offset.  If the bit is set, that means the
    /// action should be logged when it is taken, subject to the caveats
    /// described in seccomp(2).  The value of the bit vector is exposed to users
    /// in a text form in the file /proc/sys/kernel/seccomp/actions_logged.
    pub actions_logged: AtomicU16,

    /// The manager for suspend/resume.
    pub suspend_resume_manager: SuspendResumeManagerHandle,

    /// Unique IDs for new mounts and mount namespaces.
    pub next_mount_id: AtomicCounter<u64>,
    pub next_peer_group_id: AtomicCounter<u64>,
    pub next_namespace_id: AtomicCounter<u64>,

    /// Unique IDs for file objects.
    pub next_file_object_id: AtomicCounter<u64>,

    /// Controls which processes a process is allowed to ptrace.  See Documentation/security/Yama.txt
    pub ptrace_scope: AtomicU8,

    // The Fuchsia build version returned by `fuchsia.buildinfo.Provider`.
    pub build_version: OnceCell<String>,

    pub stats: Arc<KernelStats>,

    /// Resource limits that are exposed, for example, via sysctl.
    pub system_limits: SystemLimits,

    // The service to handle delayed releases. This is required for elements that requires to
    // execute some code when released and requires a known context (both in term of lock context,
    // as well as `CurrentTask`).
    pub delayed_releaser: DelayedReleaser,

    /// Manages task priorities.
    pub scheduler: SchedulerManager,

    /// The syslog manager.
    pub syslog: Syslog,

    /// All mounts.
    pub mounts: Mounts,

    /// The manager for creating and managing high-resolution timers.
    pub hrtimer_manager: HrTimerManagerHandle,

    /// The manager for monitoring and reporting resources used by the kernel.
    pub memory_attribution_manager: MemoryAttributionManager,

    /// Handler for crashing Linux processes.
    pub crash_reporter: CrashReporter,

    /// When this kernel started shutting down, if it has. See `run_shutdown()`.
    shutdown_start: OnceLock<zx::MonotonicInstant>,

    /// True to disable syslog access to unprivileged callers.  This also controls whether read
    /// access to /dev/kmsg requires privileged capabilities.
    pub restrict_dmesg: AtomicBool,

    /// Determines whether unprivileged BPF is permitted, or can be re-enabled.
    ///   0 - Unprivileged BPF is permitted.
    ///   1 - Unprivileged BPF is not permitted, and cannot be enabled.
    ///   2 - Unprivileged BPF is not permitted, but can be enabled by a privileged task.
    pub disable_unprivileged_bpf: AtomicU8,

    /// Controls use of perf_event_open(2) by tasks without CAP_PERFMON (or CAP_SYS_ADMIN):
    ///   -1 - No capability required.
    ///    0 - Capability required for tracepoint events.
    ///    1 - Capability additionally required for CPU-wide events.
    ///    2 - Capability additionally required for kernel profiling (default).
    pub perf_event_paranoid: AtomicI32,

    /// Control handle to the running container's ComponentController.
    pub container_control_handle:
        LockDepMutex<Option<ComponentControllerControlHandle>, ComponentControllerLock>,

    /// eBPF state: loaded programs, eBPF maps, etc.
    pub ebpf_state: EbpfState,

    /// Cgroups of the kernel.
    pub cgroups: KernelCgroups,

    /// Used to communicate requests to adjust system time from within a Starnix
    /// container. Used from syscalls.
    pub time_adjustment_proxy: Option<AdjustSynchronousProxy>,

    /// A token for the wake group we have registered with the netstack to
    /// receive wakeup notifications on incoming data while suspended.
    ///
    /// A value of `None` means we were unable to communicate with the netstack
    /// to create the wake group. In that case, we assume something has gone
    /// wrong with the netstack (failed to start, crash, etc) and won't retry.
    pub netstack_wake_group: OnceLock<Option<fnet_resources::WakeGroupToken>>,

    /// Used to store tokens for sockets, particularly per-uid sharing domain sockets.
    pub socket_tokens_store: SocketTokensStore,

    /// Hardware capabilities to push onto stack when loading an ELF binary.
    pub hwcaps: HwCaps,

    /// Filters for syscall logging. Processes with names matching these filters will have syscalls
    /// logged at INFO level.
    pub syscall_log_filters: LockDepMutex<Vec<SyscallLogFilter>, SyscallLogFiltersLock>,
}

/// Hardware capabilities.
#[derive(Debug, Clone, Copy, Default)]
pub struct HwCap {
    /// The value for `AT_HWCAP`.
    pub hwcap: u32,
    /// The value for `AT_HWCAP2`.
    pub hwcap2: u32,
}

/// Hardware capabilities for both 32-bit and 64-bit ELF binaries.
#[derive(Debug, Clone, Copy, Default)]
pub struct HwCaps {
    /// For 32-bit binaries.
    #[cfg(target_arch = "aarch64")]
    pub arch32: HwCap,
    /// For 64-bit binaries.
    pub arch64: HwCap,
}

/// An implementation of [`InterfacesHandler`].
///
/// This holds a `Weak<Kernel>` because it is held within a [`Netlink`] which
/// is itself held within an `Arc<Kernel>`. Holding an `Arc<T>` within an
/// `Arc<T>` prevents the `Arc`'s ref count from ever reaching 0, causing a
/// leak.
struct InterfacesHandlerImpl(Weak<Kernel>);

impl InterfacesHandlerImpl {
    fn kernel(&self) -> Option<Arc<Kernel>> {
        self.0.upgrade()
    }
}

impl InterfacesHandler for InterfacesHandlerImpl {
    fn handle_new_link(&mut self, name: &str, interface_id: NonZeroU64) {
        if let Some(kernel) = self.kernel() {
            kernel.netstack_devices.add_device(&kernel, name.into(), interface_id);
        }
    }

    fn handle_deleted_link(&mut self, name: &str) {
        if let Some(kernel) = self.kernel() {
            kernel.netstack_devices.remove_device(kernel.kthreads.system_task(), name.into());
        }
    }

    fn handle_idle_event(&mut self) {
        let Some(kernel) = self.kernel() else {
            log_error!("kernel went away while netlink is initializing");
            return;
        };
        let (initialized, wq) = &kernel.netstack_devices.initialized_and_wq;
        if initialized.swap(true, Ordering::SeqCst) {
            log_error!("netlink initial devices should only be reported once");
            return;
        }
        wq.notify_all()
    }
}

impl Kernel {
    pub fn new(
        cmdline: BString,
        features: KernelFeatures,
        system_limits: SystemLimits,
        container_namespace: ContainerNamespace,
        scheduler: SchedulerManager,
        crash_reporter_proxy: Option<CrashReporterProxy>,
        inspect_node: fuchsia_inspect::Node,
        security_state: security::KernelState,
        time_adjustment_proxy: Option<AdjustSynchronousProxy>,
        device_tree: Option<Devicetree>,
    ) -> Result<Arc<Kernel>, zx::Status> {
        let unix_address_maker =
            Box::new(|x: FsString| -> SocketAddress { SocketAddress::Unix(x) });
        let vsock_address_maker = Box::new(|x: u32| -> SocketAddress {
            SocketAddress::Vsock { port: x, cid: VMADDR_CID_HOST }
        });

        let crash_reporter = CrashReporter::new(
            &inspect_node,
            crash_reporter_proxy,
            zx::Duration::from_minutes(8),
            features.crash_report_throttling,
        );
        let hrtimer_manager = HrTimerManager::new(&inspect_node);

        let cpu_feature_flags =
            zx::system_get_feature_flags::<CpuFeatureFlags>().unwrap_or_else(|e| {
                log_debug!("CPU feature flags are only supported on ARM64: {}, reporting 0", e);
                CpuFeatureFlags::empty()
            });
        let cpu_sve_feature_flags = zx::system_get_feature_flags::<CpuSveFeatureFlags>()
            .unwrap_or_else(|e| {
                log_debug!("CPU SVE feature flags failed: {}, reporting 0", e);
                CpuSveFeatureFlags::empty()
            });
        let hwcaps = HwCaps::from_cpu_feature_flags(cpu_feature_flags, cpu_sve_feature_flags);

        let this = Arc::new_cyclic(|kernel| Kernel {
            weak_self: kernel.clone(),
            kthreads: KernelThreads::new(kernel.clone()),
            features,
            pids: Default::default(),
            init_task: OnceLock::new(),
            trace_event_manager: Arc::new(TracePerformanceEventManager::new(kernel.clone())),
            expando: Default::default(),
            default_abstract_socket_namespace: AbstractUnixSocketNamespace::new(unix_address_maker),
            default_abstract_vsock_namespace: AbstractVsockSocketNamespace::new(
                vsock_address_maker,
            ),
            cmdline,
            device_tree,
            security_state,
            device_registry: Default::default(),
            container_namespace,
            mounts_lock: RwSeqLock::new(MountsWriteToken::new().into()),
            remote_block_device_registry: Default::default(),
            iptables: OnceLock::new(),
            shared_futexes: FutexTable::default(),
            root_uts_ns: Arc::new(UtsNamespace::default().into()),
            vdso: Vdso::new(),
            vdso_arch32: Vdso::new_arch32(),
            netstack_devices: Arc::default(),
            swap_files: Default::default(),
            generic_netlink: OnceLock::new(),
            network_netlink: OnceLock::new(),
            inspect_node,
            actions_logged: AtomicU16::new(0),
            suspend_resume_manager: Default::default(),
            next_mount_id: AtomicCounter::<u64>::new(1),
            next_peer_group_id: AtomicCounter::<u64>::new(1),
            next_namespace_id: AtomicCounter::<u64>::new(1),
            next_file_object_id: Default::default(),
            system_limits,
            ptrace_scope: AtomicU8::new(0), // Disable YAMA checks by default.
            restrict_dmesg: AtomicBool::new(false),
            disable_unprivileged_bpf: AtomicU8::new(0), // Enable unprivileged BPF by default.
            perf_event_paranoid: AtomicI32::new(2),
            build_version: OnceCell::new(),
            stats: Arc::new(KernelStats::default()),
            delayed_releaser: Default::default(),
            scheduler,
            syslog: Default::default(),
            mounts: Mounts::new(),
            hrtimer_manager,
            memory_attribution_manager: MemoryAttributionManager::new(kernel.clone()),
            crash_reporter,
            shutdown_start: OnceLock::new(),
            container_control_handle: Default::default(),
            ebpf_state: Default::default(),
            cgroups: Default::default(),
            time_adjustment_proxy,
            netstack_wake_group: OnceLock::new(),
            socket_tokens_store: Default::default(),
            hwcaps,
            syscall_log_filters: Default::default(),
        });

        // Initialize the device registry before registering any devices.
        //
        // We will create sysfs recursively within this function.
        this.device_registry.objects.init(&this);

        // Make a copy of this Arc for the inspect lazy node to use but don't create an Arc cycle
        // because the inspect node that owns this reference is owned by the kernel.
        let kernel = Arc::downgrade(&this);
        this.inspect_node.record_lazy_child("thread_groups", move || {
            if let Some(kernel) = kernel.upgrade() {
                let inspector = kernel.get_thread_groups_inspect();
                async move { Ok(inspector) }.boxed()
            } else {
                async move { Err(anyhow::format_err!("kernel was dropped")) }.boxed()
            }
        });

        let kernel = Arc::downgrade(&this);
        this.inspect_node.record_lazy_child("cgroupv2", move || {
            if let Some(kernel) = kernel.upgrade() {
                async move { Ok(kernel.cgroups.cgroup2.get_cgroup_inspect()) }.boxed()
            } else {
                async move { Err(anyhow::format_err!("kernel was dropped")) }.boxed()
            }
        });

        Ok(this)
    }

    /// Returns the init task for this kernel.
    pub fn get_init_task(&self) -> Result<Arc<Task>, Errno> {
        self.init_task.get().and_then(|t| t.upgrade()).ok_or_else(|| errno!(EINVAL))
    }

    /// Shuts down userspace and the kernel in an orderly fashion, eventually terminating the root
    /// kernel process.
    pub fn shut_down(self: &Arc<Self>) {
        // Run shutdown code on a kthread in the main process so that it can be the last process
        // alive.
        self.kthreads.spawn_future(
            {
                let kernel = self.clone();
                move || async move {
                    kernel.run_shutdown().await;
                }
            },
            "run_shutdown",
        );
    }

    /// Starts shutting down the Starnix kernel and any running container. Only one thread can drive
    /// shutdown at a time. This function will return immediately if shut down is already under way.
    ///
    /// Shutdown happens in several phases:
    ///
    /// 1. Stop processes from being created, except by deferred shutdown processes until the
    ///    deferred shutdown deadline
    /// 2. Shut down individual ThreadGroups until only the init and system tasks remain, stopping
    ///    the `stop_last_on_shutdown` ones last
    /// 3. Repeat the above for the init task
    /// 4. Clean up kernel-internal structures that can hold processes alive
    /// 5. Ensure this process is the only one running in the kernel job.
    /// 6. Unmounts the kernel's mounts' FileSystems.
    /// 7. Tell CF the container component has stopped
    /// 8. Exit this process
    ///
    /// If a ThreadGroup does not shut down on its own (including after SIGKILL), that phase of
    /// shutdown will hang. To gracefully shut down any further we need the other kernel processes
    /// to do controlled exits that properly release access to shared state. If our orderly shutdown
    /// does hang, eventually CF will kill the container component which will lead to the job of
    /// this process being killed and shutdown will still complete.
    async fn run_shutdown(&self) {
        const INIT_PID: i32 = 1;
        const SYSTEM_TASK_PID: i32 = 2;

        // Step 1: From here on, only deferred shutdown processes may create processes, and only
        // until the deferred shutdown deadline (see `create_zircon_process()`). This keeps Step 2
        // finite: each round waits for its thread groups to exit, and after the deadline no new
        // ones can appear.
        if self.shutdown_start.set(zx::MonotonicInstant::get()).is_err() {
            log_info!("Additional thread tried to initiate shutdown while already in-progress.");
            return;
        }
        let deferred_shutdown_deadline =
            self.deferred_shutdown_deadline().expect("shutdown has started");

        log_info!("Shutting down Starnix kernel.");

        // Step 2: Shut down thread groups in a loop until init and the system task are all that
        // remain. Processes running a `stop_last_on_shutdown` executable are kept until everything
        // else has exited, e.g. so that processes that crash during shutdown can still have their
        // crash dumps written.
        // TODO(https://fxbug.dev/574471394): Remove once a shutdown that isn't forced lets the
        // container run its own shutdown sequence.
        let stop_last = &self.features.stop_last_on_shutdown;
        let is_stop_last = |tg: &ThreadGroup| {
            let exe = tg.read().first_task().and_then(|task| task.mm().ok());
            let exe = exe.and_then(|mm| mm.executable_node());
            exe.is_some_and(|exe| stop_last.contains(&exe.path_escaping_chroot()))
        };
        let mut keep_stop_last = !stop_last.is_empty();
        loop {
            // Collect under the pid table lock so that no process is caught while it's being
            // created. `is_stop_last()` runs after the lock is released.
            let tgs = {
                let pids = self.pids.lock();
                pids.get_thread_groups(&RcuReadScope::new())
                    .filter(|tg| tg.leader.id != SYSTEM_TASK_PID && tg.leader.id != INIT_PID)
                    .collect::<Vec<_>>()
            };
            let tgs = tgs
                .into_iter()
                .filter(|tg| !(keep_stop_last && is_stop_last(tg)))
                .collect::<Vec<_>>();
            if tgs.is_empty() {
                if keep_stop_last {
                    keep_stop_last = false;
                    continue;
                }
                log_info!("pid table is empty except init and system task");
                break;
            }

            log_info!(tgs:?; "shutting down thread groups");
            let mut tasks = vec![];
            for tg in tgs {
                let task = fasync::Task::local(ThreadGroup::shut_down(
                    Arc::downgrade(&tg),
                    deferred_shutdown_deadline,
                ));
                tasks.push(task);
            }
            futures::future::join_all(tasks).await;
        }

        // Step 3: Terminate the init process.
        let maybe_init = self.get_init_task().ok().map(|t| Arc::downgrade(&t.thread_group));
        if let Some(init) = maybe_init {
            log_info!("shutting down init");
            ThreadGroup::shut_down(init, deferred_shutdown_deadline).await;
            // If init forked while handling a core-dump signal, its children can outlive it.
            self.kill_processes_created_during_shutdown();
        } else {
            log_info!("init already terminated");
        }

        // Step 4: Clean up any structures that can keep non-Linux processes live in our job.
        log_info!("cleaning up pinned memory");
        self.expando.remove::<crate::mm::InfoCacheShadowProcess>();
        self.expando.remove::<crate::mm::MlockShadowProcess>();

        // Step 5: Make sure this is the only process running in the job. We already should have
        // cleared up all processes other than the system task at this point, but wait on any that
        // might be around for good measure.
        //
        // Use unwrap liberally since we're shutting down anyway and errors will still tear down the
        // kernel.
        let kernel_job = fuchsia_runtime::job_default();
        assert_eq!(kernel_job.children().unwrap(), &[], "starnix does not create any child jobs");
        let own_koid = fuchsia_runtime::process_self().koid().unwrap();

        log_info!("waiting for this to be the only process in the job");
        loop {
            let mut remaining_processes = kernel_job
                .processes()
                .unwrap()
                .into_iter()
                // Don't wait for ourselves to exit.
                .filter(|pid| pid != &own_koid)
                .peekable();
            if remaining_processes.peek().is_none() {
                log_info!("No stray Zircon processes.");
                break;
            }

            let mut terminated_signals = vec![];
            for pid in remaining_processes {
                let handle = match kernel_job
                    .get_child(&pid, zx::Rights::BASIC | zx::Rights::PROPERTY | zx::Rights::DESTROY)
                {
                    Ok(h) => h,
                    Err(e) => {
                        log_info!(pid:?, e:?; "failed to get child process from job");
                        continue;
                    }
                };
                log_info!(
                    pid:?,
                    name:? = handle.get_name();
                    "waiting on process terminated signal"
                );
                terminated_signals
                    .push(fuchsia_async::OnSignals::new(handle, zx::Signals::PROCESS_TERMINATED));
            }
            log_info!("waiting on process terminated signals");
            futures::future::join_all(terminated_signals).await;
        }

        // Step 6: Forcibly unmounts the mounts' FileSystems.
        log_info!("clearing mounts");
        self.mounts.clear();

        // Step 7: Tell CF the container stopped.
        log_info!("all non-root processes killed, notifying CF container is stopped");
        if let Some(control_handle) = self.container_control_handle.lock().take() {
            log_info!("Notifying CF that the container has stopped.");
            control_handle
                .send_on_stop(ComponentStopInfo {
                    termination_status: Some(zx::sys::ZX_OK),
                    exit_code: Some(0),
                    ..ComponentStopInfo::default()
                })
                .unwrap();
            control_handle.shutdown_with_epitaph(Ok(()));
        } else {
            log_warn!("Shutdown invoked without a container controller control handle.");
        }

        // Step 8: exiting this process.
        log_info!("All tasks killed, exiting Starnix kernel root process.");
        // Normally a Rust program exits its process by calling `std::process::exit()` which goes
        // through libc to exit the program. This runs drop impls on any thread-local variables
        // which can cause issues during Starnix shutdown when we haven't yet integrated every
        // subsystem with the shutdown flow. While those issues are indicative of underlying
        // problems, we can't solve them without finishing the implementation of graceful shutdown.
        // Instead, ask Zircon to exit our process directly, bypassing any libc atexit handlers.
        // TODO(https://fxbug.dev/295073633) return from main instead of avoiding atexit handlers
        zx::Process::exit(0);
    }

    pub fn is_shutting_down(&self) -> bool {
        self.shutdown_start.get().is_some()
    }

    /// Until when deferred shutdown processes may keep running and create processes, or `None` if
    /// the kernel is not shutting down. See `ThreadGroup::shut_down()`.
    pub fn deferred_shutdown_deadline(&self) -> Option<zx::MonotonicInstant> {
        self.shutdown_start.get().map(|start| *start + DEFERRED_SHUTDOWN_TIMEOUT)
    }

    /// SIGKILLs every process created during shutdown, all of which deferred shutdown processes
    /// created (e.g. crash dumpers). See `ThreadGroup::shut_down()`.
    ///
    /// A dumper that outlives the process it dumps fails, gets reported as a crash and can keep
    /// that process stopped under ptrace while it does. Processes still handling a core-dump signal
    /// aren't killed here: after their dumper exits, their handler re-raises the signal so Starnix
    /// files a crash report, the only record of a crash whose dump missed the deadline. Their own
    /// `ThreadGroup::shut_down()` gives them a moment to do so before sending SIGKILL.
    pub fn kill_processes_created_during_shutdown(&self) {
        // Collect under the pid table lock so that a process being created (which holds it from
        // the deadline check until its first task is added) is either fully added or not created.
        let created = {
            let pids = self.pids.lock();
            pids.get_thread_groups(&RcuReadScope::new())
                .filter(|tg| tg.created_during_shutdown.load(Ordering::Relaxed))
                .collect::<Vec<_>>()
        };
        for tg in created {
            tg.kill(ExitStatus::Kill(SignalInfo::kernel(SIGKILL)), None);
        }
    }

    pub fn allow_unprivileged_bpf(&self) -> bool {
        self.disable_unprivileged_bpf.load(Ordering::Relaxed) == 0
    }

    /// Opens a device file (driver) identified by `dev`.
    pub fn open_device(
        &self,
        current_task: &CurrentTask,
        node: &NamespaceNode,
        flags: OpenFlags,
        dev: DeviceId,
        mode: DeviceMode,
    ) -> Result<Box<dyn FileOps>, Errno> {
        self.device_registry.open_device(current_task, node, flags, dev, mode)
    }

    /// Return a reference to the Audit Framework
    ///
    /// This function follows the lazy initialization pattern.
    pub fn audit_logger(&self) -> Arc<AuditLogger> {
        self.expando.get_or_init(|| AuditLogger::new(self))
    }

    /// Return a reference to the GenericNetlink implementation.
    ///
    /// This function follows the lazy initialization pattern, where the first
    /// call will instantiate the Generic Netlink server in a separate kthread.
    pub fn generic_netlink(&self) -> &GenericNetlink<NetlinkToClientSender<GenericMessage>> {
        self.generic_netlink.get_or_init(|| {
            let (generic_netlink, worker_params) = GenericNetlink::new();
            let enable_nl80211 = self.features.wifi;
            self.kthreads.spawn_future(
                move || async move {
                    crate::vfs::socket::run_generic_netlink_worker(worker_params, enable_nl80211)
                        .await;
                    log_error!("Generic Netlink future unexpectedly exited");
                },
                "generic_netlink_worker",
            );
            generic_netlink
        })
    }

    /// Return a reference to the [`netlink::Netlink`] implementation.
    ///
    /// This function follows the lazy initialization pattern, where the first
    /// call will instantiate the Netlink implementation.
    pub fn network_netlink(self: &Arc<Self>) -> &Netlink<NetlinkContextImpl> {
        self.network_netlink.get_or_init(|| {
            let (network_netlink, worker_params) =
                Netlink::new(InterfacesHandlerImpl(self.weak_self.clone()));

            let kernel = self.clone();
            self.kthreads.spawn_future(
                move || async move {
                    netlink::run_netlink_worker(
                        worker_params,
                        NetlinkAccessControl::new(kernel.kthreads.system_task()),
                    )
                    .await;
                    log_error!(tag = NETLINK_LOG_TAG; "Netlink async worker unexpectedly exited");
                },
                "network_netlink_worker",
            );
            network_netlink
        })
    }

    /// Return a reference to the token representing our wake group registered
    /// with the netstack.
    ///
    /// This function lazily initializes the wake group with the netstack and
    /// the wake group as a wake source with the Starnix runner.
    pub(crate) fn netstack_wake_group(&self) -> Option<&fnet_resources::WakeGroupToken> {
        self.netstack_wake_group
            .get_or_init(|| {
                // The signal the netstack raises when it wants the container to
                // wake up.
                const GROUP_WAKEUP_SIGNAL: zx::Signals =
                    zx::Signals::from_bits(fnet_power::GROUP_WAKEUP_SIGNAL).unwrap();

                let provider = fuchsia_component::client::connect_to_protocol_sync::<
                    fnet_power::WakeGroupProviderMarker,
                >()
                .expect("connect to WakeGroupProvider");

                let (wake_watcher_waiter, wake_watcher_signaller) = zx::EventPair::create();
                create_watcher_for_wake_events(wake_watcher_signaller);

                let token = match provider.create_wake_group(
                    &fnet_power::WakeGroupOptions {
                        debug_name: Some(String::from("starnix")),
                        ..Default::default()
                    },
                    wake_watcher_waiter,
                    zx::Instant::INFINITE,
                ) {
                    Ok(fnet_power::CreateWakeGroupResponse { token, .. }) => token,
                    Err(e) => {
                        log_error!("failed to create wake group with the netstack: {e}");
                        return None;
                    }
                };

                let token = token.expect("netstack provides a wake group token");
                let wake_source = token
                    .token
                    .duplicate_handle(zx::Rights::SAME_RIGHTS)
                    .expect("duplicate handle to token");
                self.suspend_resume_manager
                    .add_external_wake_source(
                        wake_source.into(),
                        GROUP_WAKEUP_SIGNAL,
                        "netstack-wake-group".into(),
                    )
                    .expect("add wake group as wake source");

                Some(token)
            })
            .as_ref()
    }

    pub fn iptables(&self) -> &IpTables {
        self.iptables.get_or_init(|| IpTables::new())
    }

    /// Returns a Proxy to the service used by the container at `filename`.
    #[allow(unused)]
    pub fn connect_to_named_protocol_at_container_svc<P: ProtocolMarker>(
        &self,
        filename: &str,
    ) -> Result<ClientEnd<P>, Errno> {
        match self.container_namespace.get_namespace_channel("/svc") {
            Ok(channel) => {
                let (client_end, server_end) = create_endpoints::<P>();
                fdio::service_connect_at(channel.as_ref(), filename, server_end.into_channel())
                    .map_err(|status| from_status_like_fdio!(status))?;
                Ok(client_end)
            }
            Err(err) => {
                log_error!("Unable to get /svc namespace channel! {}", err);
                Err(errno!(ENOENT))
            }
        }
    }

    /// Returns a Proxy to the service `P` used by the container.
    pub fn connect_to_protocol_at_container_svc<P: DiscoverableProtocolMarker>(
        &self,
    ) -> Result<ClientEnd<P>, Errno> {
        self.connect_to_named_protocol_at_container_svc::<P>(P::PROTOCOL_NAME)
    }

    pub fn add_syscall_log_filter(&self, name: &str) {
        let filter = SyscallLogFilter::new(name.to_string());
        {
            let mut filters = self.syscall_log_filters.lock();
            if filters.contains(&filter) {
                return;
            }
            filters.push(filter);
        }
        for headers in self.pids.get_thread_groups(&RcuReadScope::new()) {
            headers.sync_syscall_log_level();
        }
    }

    pub fn clear_syscall_log_filters(&self) {
        {
            let mut filters = self.syscall_log_filters.lock();
            if filters.is_empty() {
                return;
            }
            filters.clear();
        }
        for headers in self.pids.get_thread_groups(&RcuReadScope::new()) {
            headers.sync_syscall_log_level();
        }
    }

    fn get_thread_groups_inspect(&self) -> fuchsia_inspect::Inspector {
        let inspector = fuchsia_inspect::Inspector::default();

        let thread_groups = inspector.root();
        let mut mm_summary = MappingSummary::default();
        let mut mms_summarized = HashSet::new();

        for thread_group in self.pids.get_thread_groups(&RcuReadScope::new()) {
            // Avoid holding the state lock while summarizing.
            let (ppid, tasks) = {
                let tg = thread_group.read();
                (tg.get_ppid() as i64, tg.tasks())
            };

            let tg_node = thread_groups.create_child(format!("{}", thread_group.leader));
            if let Ok(koid) = thread_group.process.koid() {
                tg_node.record_int("koid", koid.raw_koid() as i64);
            }
            tg_node.record_int("pid", thread_group.leader.id as i64);
            tg_node.record_int("ppid", ppid);
            tg_node.record_bool("stopped", thread_group.load_stopped() == StopState::GroupStopped);

            let tasks_node = tg_node.create_child("tasks");
            for task in tasks {
                if let Ok(mm) = task.mm() {
                    if mms_summarized.insert(Arc::as_ptr(&mm) as usize) {
                        mm.summarize(&mut mm_summary);
                    }
                }
                let set_properties = |node: &fuchsia_inspect::Node| {
                    node.record_string("command", task.command().to_string());

                    let scheduler_state = task.scheduler_state.lock().clone();
                    if !scheduler_state.is_default() {
                        node.record_child("sched", |node| {
                            node.record_string(
                                "role_name",
                                self.scheduler
                                    .role_name(&task, &scheduler_state)
                                    .map(|n| Cow::Borrowed(n))
                                    .unwrap_or_else(|e| Cow::Owned(e.to_string())),
                            );
                            node.record_string("state", format!("{scheduler_state:?}"));
                        });
                    }
                };
                if task.tid == thread_group.leader {
                    let mut argv = task.read_argv(256).unwrap_or_default();

                    // Any runtime that overwrites argv is likely to leave a lot of trailing
                    // nulls, no need to print those in inspect.
                    argv.retain(|arg| !arg.is_empty());

                    let inspect_argv = tg_node.create_string_array("argv", argv.len());
                    for (i, arg) in argv.iter().enumerate() {
                        inspect_argv.set(i, arg.to_string());
                    }
                    tg_node.record(inspect_argv);

                    set_properties(&tg_node);
                } else {
                    tasks_node.record_child(task.tid.to_string(), |task_node| {
                        set_properties(task_node);
                    });
                };
            }
            tg_node.record(tasks_node);
            thread_groups.record(tg_node);
        }

        thread_groups.record_child("memory_managers", |node| mm_summary.record(node));

        inspector
    }

    pub fn new_memory_attribution_observer(
        &self,
        control_handle: fattribution::ProviderControlHandle,
    ) -> attribution_server::Observer {
        self.memory_attribution_manager.new_observer(control_handle)
    }

    /// Opens and returns a directory proxy from the container's namespace, at
    /// the requested path, using the provided flags. This method will open the
    /// closest existing path from the namespace hierarchy, and then attempt
    /// initialize an open on the remaining subdirectory path, using the given open_flags.
    ///
    /// For example, given the parameter provided is `/path/to/foo/bar` and there
    /// are namespace entries already for `/path/to/foo` and `/path/to`. The entry
    /// for /path/to/foo will be opened, and then the /bar will attempt to be opened
    /// underneath that directory with the given open_flags. The returned value
    /// will be the proxy to the parent (/path/to/foo) and the string to the child
    /// path (/bar). The caller of this method can expect /bar to be initialized.
    pub fn open_ns_dir(
        &self,
        path: &str,
        open_flags: fio::Flags,
    ) -> Result<(fio::DirectorySynchronousProxy, String), Errno> {
        let ns_path = PathBuf::from(path);
        match self.container_namespace.find_closest_channel(&ns_path) {
            Ok((root_channel, remaining_subdir)) => {
                let (_, server_end) = create_endpoints::<fio::DirectoryMarker>();
                fdio::open_at(
                    &root_channel,
                    &remaining_subdir,
                    open_flags,
                    server_end.into_channel(),
                )
                .map_err(|e| {
                    log_error!("Failed to intialize the subdirs: {}", e);
                    errno!(EIO)
                })?;

                Ok((fio::DirectorySynchronousProxy::new(root_channel), remaining_subdir))
            }
            Err(err) => {
                log_error!(
                    "Unable to find a channel for {}. Received error: {}",
                    ns_path.display(),
                    err
                );
                Err(errno!(ENOENT))
            }
        }
    }

    /// Returns an iterator of the command line arguments.
    pub fn cmdline_args_iter(&self) -> impl Iterator<Item = ArgNameAndValue<'_>> {
        parse_cmdline(self.cmdline.to_str().unwrap_or_default()).map(ArgNameAndValue::from)
    }

    /// Returns the container-configured CacheConfig.
    pub fn fs_cache_config(&self) -> CacheConfig {
        CacheConfig { capacity: self.features.dirent_cache_size as usize }
    }

    pub fn mounts_lock(&self) -> MountsWriteGuard<'_> {
        MountsWriteGuard::new(self.mounts_lock.lock())
    }
}

pub fn parse_cmdline(cmdline: &str) -> impl Iterator<Item = &str> {
    let mut args = Vec::new();
    let mut arg_start: Option<usize> = None;
    let mut in_quotes = false;
    let mut previous_char = ' ';

    for (i, c) in cmdline.char_indices() {
        if let Some(start) = arg_start {
            match c {
                ' ' if !in_quotes => {
                    args.push(&cmdline[start..i]);
                    arg_start = None;
                }
                '"' if previous_char != '\\' => {
                    in_quotes = !in_quotes;
                }
                _ => {}
            }
        } else if c != ' ' {
            arg_start = Some(i);
            if c == '"' {
                in_quotes = true;
            }
        }
        previous_char = c;
    }
    if let Some(start) = arg_start {
        args.push(&cmdline[start..]);
    }
    args.into_iter()
}

impl std::fmt::Debug for Kernel {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Kernel").finish()
    }
}

// TODO(https://fxbug.dev/380427153): move arch dependent code to `kernel/core/arch/*`.
#[cfg(target_arch = "aarch64")]
fn arm32_hwcap(cpu_feature_flags: CpuFeatureFlags) -> HwCap {
    use starnix_uapi::arch32;
    const COMPAT_ARM32_ELF_HWCAP: u32 = arch32::HWCAP_HALF
        | arch32::HWCAP_THUMB
        | arch32::HWCAP_FAST_MULT
        | arch32::HWCAP_EDSP
        | arch32::HWCAP_TLS
        | arch32::HWCAP_IDIV // == IDIVA | IDIVT.
        | arch32::HWCAP_LPAE
        | arch32::HWCAP_EVTSTRM;

    let mut hwcap = COMPAT_ARM32_ELF_HWCAP;
    let mut hwcap2 = 0;
    for feature in cpu_feature_flags.iter() {
        match feature {
            CpuFeatureFlags::ARM64_FEATURE_ISA_ASIMD => hwcap |= arch32::HWCAP_NEON,
            CpuFeatureFlags::ARM64_FEATURE_ISA_AES => hwcap2 |= arch32::HWCAP2_AES,
            CpuFeatureFlags::ARM64_FEATURE_ISA_PMULL => hwcap2 |= arch32::HWCAP2_PMULL,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SHA1 => hwcap2 |= arch32::HWCAP2_SHA1,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SHA256 => hwcap2 |= arch32::HWCAP2_SHA2,
            CpuFeatureFlags::ARM64_FEATURE_ISA_CRC32 => hwcap2 |= arch32::HWCAP2_CRC32,
            CpuFeatureFlags::ARM64_FEATURE_ISA_I8MM => hwcap |= arch32::HWCAP_I8MM,
            CpuFeatureFlags::ARM64_FEATURE_ISA_FHM => hwcap |= arch32::HWCAP_ASIMDFHM,
            CpuFeatureFlags::ARM64_FEATURE_ISA_DP => hwcap |= arch32::HWCAP_ASIMDDP,
            CpuFeatureFlags::ARM64_FEATURE_ISA_FP => {
                hwcap |= arch32::HWCAP_VFP | arch32::HWCAP_VFPv3 | arch32::HWCAP_VFPv4
            }
            _ => {}
        }
    }
    // Both FEAT_FHM and FEAT_SVE architecturally require FEAT_FP16 (which is defined for both
    // AArch64 and AArch32 execution states).
    // TODO(https://fxbug.dev/42081850): Expose a dedicated `ZX_ARM64_FEATURE_ISA_FP16` flag from
    // Zircon so standalone Armv8.2 FEAT_FP16 cores without FHM or SVE also report FP16 hwcaps.
    if cpu_feature_flags
        .intersects(CpuFeatureFlags::ARM64_FEATURE_ISA_FHM | CpuFeatureFlags::ARM64_FEATURE_ISA_SVE)
    {
        hwcap |= arch32::HWCAP_FPHP | arch32::HWCAP_ASIMDHP;
    }
    HwCap { hwcap, hwcap2 }
}

#[cfg(target_arch = "aarch64")]
fn arm64_hwcap(
    cpu_feature_flags: CpuFeatureFlags,
    cpu_sve_feature_flags: CpuSveFeatureFlags,
) -> HwCap {
    // See https://docs.kernel.org/arch/arm64/elf_hwcaps.html for details.
    use starnix_uapi;
    let mut hwcap = 0;
    let mut hwcap2 = 0;

    for feature in cpu_feature_flags.iter() {
        match feature {
            CpuFeatureFlags::ARM64_FEATURE_ISA_FP => hwcap |= starnix_uapi::HWCAP_FP,
            CpuFeatureFlags::ARM64_FEATURE_ISA_ASIMD => hwcap |= starnix_uapi::HWCAP_ASIMD,
            CpuFeatureFlags::ARM64_FEATURE_ISA_AES => hwcap |= starnix_uapi::HWCAP_AES,
            CpuFeatureFlags::ARM64_FEATURE_ISA_PMULL => hwcap |= starnix_uapi::HWCAP_PMULL,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SHA1 => hwcap |= starnix_uapi::HWCAP_SHA1,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SHA256 => hwcap |= starnix_uapi::HWCAP_SHA2,
            CpuFeatureFlags::ARM64_FEATURE_ISA_CRC32 => hwcap |= starnix_uapi::HWCAP_CRC32,
            CpuFeatureFlags::ARM64_FEATURE_ISA_I8MM => hwcap2 |= starnix_uapi::HWCAP2_I8MM,
            CpuFeatureFlags::ARM64_FEATURE_ISA_FHM => hwcap |= starnix_uapi::HWCAP_ASIMDFHM,
            CpuFeatureFlags::ARM64_FEATURE_ISA_DP => hwcap |= starnix_uapi::HWCAP_ASIMDDP,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SM3 => hwcap |= starnix_uapi::HWCAP_SM3,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SM4 => hwcap |= starnix_uapi::HWCAP_SM4,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SHA3 => hwcap |= starnix_uapi::HWCAP_SHA3,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SHA512 => hwcap |= starnix_uapi::HWCAP_SHA512,
            CpuFeatureFlags::ARM64_FEATURE_ISA_ATOMICS => hwcap |= starnix_uapi::HWCAP_ATOMICS,
            CpuFeatureFlags::ARM64_FEATURE_ISA_RDM => hwcap |= starnix_uapi::HWCAP_ASIMDRDM,
            CpuFeatureFlags::ARM64_FEATURE_ISA_TS => hwcap |= starnix_uapi::HWCAP_FLAGM,
            CpuFeatureFlags::ARM64_FEATURE_ISA_DPB => hwcap |= starnix_uapi::HWCAP_DCPOP,
            CpuFeatureFlags::ARM64_FEATURE_ISA_RNDR => hwcap2 |= starnix_uapi::HWCAP2_RNG,
            CpuFeatureFlags::ARM64_FEATURE_ISA_SVE => hwcap |= starnix_uapi::HWCAP_SVE,
            _ => {}
        }
    }
    // Both FEAT_FHM and FEAT_SVE architecturally require FEAT_FP16 (HWCAP_FPHP and HWCAP_ASIMDHP).
    // Per ACLE Function Multi-Versioning (FMV) dependency rules (used by LLVM/compiler-rt), "sve"
    // and "sve2" targets transitively require FEAT_FP | FEAT_FP16 | FEAT_SVE, where compiler-rt
    // derives FEAT_FP16 from HWCAP_FPHP.
    // TODO(https://fxbug.dev/42081850): Expose a dedicated `ZX_ARM64_FEATURE_ISA_FP16` flag from
    // Zircon so standalone Armv8.2 FEAT_FP16 cores without FHM or SVE also report FP16 hwcaps.
    if cpu_feature_flags
        .intersects(CpuFeatureFlags::ARM64_FEATURE_ISA_FHM | CpuFeatureFlags::ARM64_FEATURE_ISA_SVE)
    {
        hwcap |= starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP;
    }
    if cpu_feature_flags.contains(CpuFeatureFlags::ARM64_FEATURE_ISA_SVE) {
        for feature in cpu_sve_feature_flags.iter() {
            match feature {
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEI8MM => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEI8MM
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF32MM => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEF32MM
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF64MM => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEF64MM
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBF16 => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEBF16
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVE2 => hwcap2 |= starnix_uapi::HWCAP2_SVE2,
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEAES => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEAES
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEPMULL => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEPMULL
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBITPERM => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVEBITPERM
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESHA3 => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVESHA3
                }
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESM4 => {
                    hwcap2 |= starnix_uapi::HWCAP2_SVESM4
                }
                _ => {}
            }
        }
    }
    HwCap { hwcap, hwcap2 }
}

impl HwCaps {
    #[cfg(target_arch = "aarch64")]
    pub fn from_cpu_feature_flags(
        cpu_feature_flags: CpuFeatureFlags,
        cpu_sve_feature_flags: CpuSveFeatureFlags,
    ) -> Self {
        Self {
            arch32: arm32_hwcap(cpu_feature_flags),
            arch64: arm64_hwcap(cpu_feature_flags, cpu_sve_feature_flags),
        }
    }

    #[cfg(not(target_arch = "aarch64"))]
    pub fn from_cpu_feature_flags(
        _cpu_feature_flags: CpuFeatureFlags,
        _cpu_sve_feature_flags: CpuSveFeatureFlags,
    ) -> Self {
        Self { arch64: HwCap::default() }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_parse_cmdline() {
        let cmdline =
            r#"first second=third "fourth fifth" sixth="seventh eighth" "ninth\" tenth" eleventh"#;
        let expected = vec![
            "first",
            "second=third",
            "\"fourth fifth\"",
            "sixth=\"seventh eighth\"",
            "\"ninth\\\" tenth\"",
            "eleventh",
        ];
        assert_eq!(parse_cmdline(cmdline).collect::<Vec<_>>(), expected);
    }

    #[cfg(target_arch = "aarch64")]
    #[test]
    fn test_arm64_hwcap_sve() {
        let empty =
            HwCaps::from_cpu_feature_flags(CpuFeatureFlags::empty(), CpuSveFeatureFlags::empty());
        assert_eq!(empty.arch64.hwcap, 0);
        assert_eq!(empty.arch64.hwcap2, 0);

        let expected_sve_hwcap =
            starnix_uapi::HWCAP_SVE | starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP;
        let sve_only = HwCaps::from_cpu_feature_flags(
            CpuFeatureFlags::ARM64_FEATURE_ISA_SVE,
            CpuSveFeatureFlags::empty(),
        );
        assert_eq!(sve_only.arch64.hwcap, expected_sve_hwcap);
        assert_eq!(sve_only.arch64.hwcap2, 0);
        assert_eq!(
            sve_only.arch32.hwcap,
            empty.arch32.hwcap
                | starnix_uapi::arch32::HWCAP_FPHP
                | starnix_uapi::arch32::HWCAP_ASIMDHP
        );
        assert_eq!(sve_only.arch32.hwcap2, 0);

        let fhm_only = HwCaps::from_cpu_feature_flags(
            CpuFeatureFlags::ARM64_FEATURE_ISA_FHM,
            CpuSveFeatureFlags::empty(),
        );
        assert_eq!(
            fhm_only.arch64.hwcap,
            starnix_uapi::HWCAP_ASIMDFHM | starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP
        );
        assert_eq!(fhm_only.arch64.hwcap2, 0);
        assert_eq!(
            fhm_only.arch32.hwcap,
            empty.arch32.hwcap
                | starnix_uapi::arch32::HWCAP_ASIMDFHM
                | starnix_uapi::arch32::HWCAP_FPHP
                | starnix_uapi::arch32::HWCAP_ASIMDHP
        );
        assert_eq!(fhm_only.arch32.hwcap2, 0);

        let cases = [
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEI8MM, starnix_uapi::HWCAP2_SVEI8MM),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF32MM, starnix_uapi::HWCAP2_SVEF32MM),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF64MM, starnix_uapi::HWCAP2_SVEF64MM),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBF16, starnix_uapi::HWCAP2_SVEBF16),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVE2, starnix_uapi::HWCAP2_SVE2),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEAES, starnix_uapi::HWCAP2_SVEAES),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEPMULL, starnix_uapi::HWCAP2_SVEPMULL),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBITPERM, starnix_uapi::HWCAP2_SVEBITPERM),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESHA3, starnix_uapi::HWCAP2_SVESHA3),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESM4, starnix_uapi::HWCAP2_SVESM4),
        ];

        let mut all_sve_flags = CpuSveFeatureFlags::empty();
        let mut expected_hwcap2 = 0;
        for (flag, hwcap2_bit) in cases {
            let without_sve = HwCaps::from_cpu_feature_flags(CpuFeatureFlags::empty(), flag);
            assert_eq!(without_sve.arch64.hwcap, 0);
            assert_eq!(without_sve.arch64.hwcap2, 0);

            let caps = HwCaps::from_cpu_feature_flags(CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, flag);
            assert_eq!(caps.arch64.hwcap, expected_sve_hwcap);
            assert_eq!(caps.arch64.hwcap2, hwcap2_bit);
            all_sve_flags |= flag;
            expected_hwcap2 |= hwcap2_bit;
        }

        let all_caps =
            HwCaps::from_cpu_feature_flags(CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, all_sve_flags);
        assert_eq!(all_caps.arch64.hwcap, expected_sve_hwcap);
        assert_eq!(all_caps.arch64.hwcap2, expected_hwcap2);
    }

    #[cfg(target_arch = "aarch64")]
    #[test]
    fn test_arm64_hwcap_sve_gating_and_linux_abi() {
        // The HWCAP bit positions are part of the Linux ABI
        // (`arch/arm64/include/uapi/asm/hwcap.h` and `arch/arm/include/uapi/asm/hwcap.h`). Pin
        // them, so that a wrong uapi binding or a transposed mapping in `arm64_hwcap` /
        // `arm32_hwcap` is caught immediately.
        assert_eq!(starnix_uapi::HWCAP_FP, 1 << 0);
        assert_eq!(starnix_uapi::HWCAP_ASIMD, 1 << 1);
        assert_eq!(starnix_uapi::HWCAP_FPHP, 1 << 9);
        assert_eq!(starnix_uapi::HWCAP_ASIMDHP, 1 << 10);
        assert_eq!(starnix_uapi::HWCAP_SVE, 1 << 22);
        assert_eq!(starnix_uapi::HWCAP_ASIMDFHM, 1 << 23);
        assert_eq!(starnix_uapi::arch32::HWCAP_FPHP, 1 << 22);
        assert_eq!(starnix_uapi::arch32::HWCAP_ASIMDHP, 1 << 23);
        assert_eq!(starnix_uapi::arch32::HWCAP_ASIMDFHM, 1 << 25);
        let expected_sve_hwcap =
            starnix_uapi::HWCAP_SVE | starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP;
        let sve_mappings = [
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVE2, starnix_uapi::HWCAP2_SVE2, 1 << 1),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEAES, starnix_uapi::HWCAP2_SVEAES, 1 << 2),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEPMULL, starnix_uapi::HWCAP2_SVEPMULL, 1 << 3),
            (
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBITPERM,
                starnix_uapi::HWCAP2_SVEBITPERM,
                1 << 4,
            ),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESHA3, starnix_uapi::HWCAP2_SVESHA3, 1 << 5),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESM4, starnix_uapi::HWCAP2_SVESM4, 1 << 6),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEI8MM, starnix_uapi::HWCAP2_SVEI8MM, 1 << 9),
            (
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF32MM,
                starnix_uapi::HWCAP2_SVEF32MM,
                1 << 10,
            ),
            (
                CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF64MM,
                starnix_uapi::HWCAP2_SVEF64MM,
                1 << 11,
            ),
            (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBF16, starnix_uapi::HWCAP2_SVEBF16, 1 << 12),
        ];
        let mut all_sve_flags = CpuSveFeatureFlags::empty();
        let mut all_sve_hwcap2 = 0u32;
        for (flag, hwcap2_bit, linux_bit) in sve_mappings {
            assert_eq!(hwcap2_bit, linux_bit, "{flag:?}");
            assert_eq!(all_sve_hwcap2 & hwcap2_bit, 0, "{flag:?} HWCAP2 bit is not unique");
            all_sve_flags |= flag;
            all_sve_hwcap2 |= hwcap2_bit;
        }
        // Every named SVE subfeature reported by Zircon has a mapping.
        let all_named_sve_flags = <CpuSveFeatureFlags as bitflags::Flags>::FLAGS
            .iter()
            .filter(|f| f.is_named())
            .fold(CpuSveFeatureFlags::empty(), |acc, f| acc | *f.value());
        assert_eq!(all_sve_flags, all_named_sve_flags);

        // Linux never reports HWCAP2_SVE* without HWCAP_SVE. Without ZX_ARM64_FEATURE_ISA_SVE, the
        // SVE subfeatures (all of them at once, or even unknown bits in ZX_FEATURE_KIND_CPU_SVE)
        // must be ignored, while unrelated hwcaps are unaffected.
        let non_sve_cpu_flags = CpuFeatureFlags::ARM64_FEATURE_ISA_FP
            | CpuFeatureFlags::ARM64_FEATURE_ISA_ASIMD
            | CpuFeatureFlags::ARM64_FEATURE_ISA_I8MM
            | CpuFeatureFlags::ARM64_FEATURE_ISA_RNDR;
        let non_sve_hwcap = starnix_uapi::HWCAP_FP | starnix_uapi::HWCAP_ASIMD;
        let non_sve_hwcap2 = starnix_uapi::HWCAP2_I8MM | starnix_uapi::HWCAP2_RNG;
        let baseline =
            HwCaps::from_cpu_feature_flags(non_sve_cpu_flags, CpuSveFeatureFlags::empty());
        assert_eq!(baseline.arch64.hwcap, non_sve_hwcap);
        assert_eq!(baseline.arch64.hwcap2, non_sve_hwcap2);
        for sve_flags in [all_sve_flags, CpuSveFeatureFlags::from_bits_retain(!0)] {
            let gated = HwCaps::from_cpu_feature_flags(CpuFeatureFlags::empty(), sve_flags);
            assert_eq!(gated.arch64.hwcap, 0, "{sve_flags:?}");
            assert_eq!(gated.arch64.hwcap2, 0, "{sve_flags:?}");

            let gated = HwCaps::from_cpu_feature_flags(non_sve_cpu_flags, sve_flags);
            assert_eq!(gated.arch64.hwcap, non_sve_hwcap, "{sve_flags:?}");
            assert_eq!(gated.arch64.hwcap & starnix_uapi::HWCAP_SVE, 0, "{sve_flags:?}");
            assert_eq!(gated.arch64.hwcap2, non_sve_hwcap2, "{sve_flags:?}");
            assert_eq!(gated.arch64.hwcap2 & all_sve_hwcap2, 0, "{sve_flags:?}");
            // The 32-bit (compat) hwcaps never include SVE.
            assert_eq!(gated.arch32.hwcap, baseline.arch32.hwcap);
            assert_eq!(gated.arch32.hwcap2, baseline.arch32.hwcap2);

            // With ZX_ARM64_FEATURE_ISA_SVE, exactly the known subfeature bits are added on top of
            // the unrelated hwcaps, and unknown bits in ZX_FEATURE_KIND_CPU_SVE are still ignored.
            let enabled = HwCaps::from_cpu_feature_flags(
                non_sve_cpu_flags | CpuFeatureFlags::ARM64_FEATURE_ISA_SVE,
                sve_flags,
            );
            assert_eq!(enabled.arch64.hwcap, non_sve_hwcap | expected_sve_hwcap);
            assert_eq!(enabled.arch64.hwcap2, non_sve_hwcap2 | all_sve_hwcap2);
            assert_eq!(
                enabled.arch32.hwcap,
                baseline.arch32.hwcap
                    | starnix_uapi::arch32::HWCAP_FPHP
                    | starnix_uapi::arch32::HWCAP_ASIMDHP
            );
            assert_eq!(enabled.arch32.hwcap2, baseline.arch32.hwcap2);
        }

        // Leaving out a single subfeature clears exactly its HWCAP2 bit, which catches duplicated
        // or transposed mappings that `test_arm64_hwcap_sve`'s single-flag checks would not.
        for (flag, hwcap2_bit, _) in sve_mappings {
            let all_but_one = HwCaps::from_cpu_feature_flags(
                CpuFeatureFlags::ARM64_FEATURE_ISA_SVE,
                all_sve_flags.difference(flag),
            );
            assert_eq!(all_but_one.arch64.hwcap, expected_sve_hwcap, "{flag:?}");
            assert_eq!(all_but_one.arch64.hwcap2, all_sve_hwcap2 & !hwcap2_bit, "{flag:?}");
        }
    }
}