Skip to content

Commit

Permalink
feat: Refactor init into test-distro
Browse files Browse the repository at this point in the history
The init module contains a small init system for running our integration
tests against a kernel. While we don't need a full-blown linux distro,
we do need some utilities.

Once such utility is `modprobe` which allows us to load kernel modules.
Rather than create a new module for this utility, I've instead
refactored `init` into `test-distro` which is a module that contains
multiple binaries.

The xtask code has been adjusted to ensure these binaries are inserted
into the correct places in our cpio archive, as well as bringing in the
kernel modules.

Signed-off-by: Dave Tucker <[email protected]>
  • Loading branch information
dave-tucker committed Feb 2, 2025
1 parent 921e457 commit bc64c61
Show file tree
Hide file tree
Showing 9 changed files with 231 additions and 56 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ jobs:
run: |
set -euxo pipefail
find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' '**/modules/*' --file -"
- name: Run local integration tests
if: runner.os == 'Linux'
Expand Down
6 changes: 4 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ members = [
"aya-log-parser",
"aya-obj",
"aya-tool",
"init",
"test-distro",
"test/integration-common",
"test/integration-test",
"xtask",
Expand All @@ -33,7 +33,7 @@ default-members = [
"aya-log-parser",
"aya-obj",
"aya-tool",
"init",
"test-distro",
"test/integration-common",
# test/integration-test is omitted; including it in this list causes `cargo test` to run its
# tests, and that doesn't work unless they've been built with `cargo xtask`.
Expand Down Expand Up @@ -73,6 +73,7 @@ diff = { version = "0.1.13", default-features = false }
env_logger = { version = "0.11", default-features = false }
epoll = { version = "4.3.3", default-features = false }
futures = { version = "0.3.28", default-features = false }
glob = { version = "0.3.0", default-features = false }
hashbrown = { version = "0.15.0", default-features = false }
indoc = { version = "2.0", default-features = false }
libc = { version = "0.2.105", default-features = false }
Expand Down Expand Up @@ -101,6 +102,7 @@ thiserror = { version = "2.0.3", default-features = false }
tokio = { version = "1.24.0", default-features = false }
which = { version = "7.0.0", default-features = false }
xdpilone = { version = "1.0.5", default-features = false }
xz2 = { version = "0.1.7", default-features = false }

[profile.release.package.integration-ebpf]
debug = 2
Expand Down
13 changes: 0 additions & 13 deletions init/Cargo.toml

This file was deleted.

24 changes: 24 additions & 0 deletions test-distro/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[package]
name = "test-distro"
version = "0.1.0"
publish = false
authors.workspace = true
license.workspace = true
repository.workspace = true
homepage.workspace = true
edition.workspace = true

[[bin]]
name = "init"
path = "src/init.rs"

[[bin]]
name = "modprobe"
path = "src/modprobe.rs"

[dependencies]
anyhow = { workspace = true, features = ["std"] }
clap = { workspace = true, default-features = true, features = ["derive"] }
nix = { workspace = true, features = ["fs", "mount", "reboot", "kmod"] }
glob = { workspace = true }
xz2 = { workspace = true }
1 change: 1 addition & 0 deletions init/src/main.rs → test-distro/src/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ fn run() -> anyhow::Result<()> {
let path = entry.path();
let status = std::process::Command::new(&path)
.args(&args)
.env("PATH", "/sbin:/usr/sbin:/bin:/usr/bin")
.env("RUST_LOG", "debug")
.status()
.with_context(|| format!("failed to execute {}", path.display()))?;
Expand Down
79 changes: 79 additions & 0 deletions test-distro/src/modprobe.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
//! modprobe is used to load kernel modules into the kernel.
//!
//! This implementation is incredibly naive and is only designed to work within
//! the constraints of the test environment. Not for production use.
use std::{ffi::CString, io::Read as _};

use clap::Parser;
use glob::glob;
use nix::kmod::init_module;

#[derive(Parser)]
struct Args {
/// Suppress all output
#[clap(short, long, default_value = "false")]
quiet: bool,

/// Provide the path to the modules directory for testing
#[clap(long, default_value = "/lib/modules")]
modules_dir: String,
name: String,
}

fn main() {
let args = Args::parse();

// This modprobe only loads modules.
// It also doesn't handle dependencies.

let module = &args.name;
let pattern = format!("{}/**/{}.k*", args.modules_dir, module);

let matches = glob(&pattern)
.expect("Failed to read glob pattern")
.filter_map(Result::ok)
.collect::<Vec<_>>();

if matches.is_empty() {
if !args.quiet {
eprintln!("No module found for pattern: {}", pattern);
}
std::process::exit(1);
}

let path = &matches[0];

if !args.quiet {
println!("Loading module: {}", path.display());
}
let mut f = std::fs::File::open(path).expect("Failed to open module file");
let mut contents: Vec<u8> = Vec::new();
f.read_to_end(&mut contents).unwrap();

if let Some(extension) = path.extension() {
if extension == "xz" {
if !args.quiet {
println!("Decompressing module");
}
let mut decompressed = Vec::new();
xz2::read::XzDecoder::new(&contents[..])
.read_to_end(&mut decompressed)
.expect("Failed to decompress module");
contents = decompressed;
}
}

if contents[0..4] != [0x7f, 0x45, 0x4c, 0x46] {
if !args.quiet {
eprintln!("Module is not an valid ELF file");
}
std::process::exit(1);
}

init_module(&contents, &CString::new("").unwrap()).expect("Failed to load module into kernel");

if !args.quiet {
println!("Module loaded successfully");
}
}
1 change: 1 addition & 0 deletions test/integration-test/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ mod info;
mod iter;
mod load;
mod log;
mod modprobe;
mod raw_tracepoint;
mod rbpf;
mod relocations;
Expand Down
28 changes: 28 additions & 0 deletions test/integration-test/src/tests/modprobe.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
use std::{path::Path, process::Command};

use aya::programs::tc;

use crate::utils::NetNsGuard;

// FIXME: Delete this test before merging.
// This is used to test the modprobe command inside the vm.
#[test]
fn modprobe() {
println!("PATH: {:?}", std::env::var("PATH").unwrap());

// check modprobe exists
assert!(Path::new("/sbin/modprobe").exists());

// This operation requires the kernel to load
// the sch_ingress module. If module auto-loading
// is working correctly, this should succeed...
// However, I haven't cracked that nut yet, so for now...
let _ = Command::new("modprobe")
.arg("sch_ingress")
.status()
.expect("failed to execute modprobe");

let _netns = NetNsGuard::new();

tc::qdisc_add_clsact("lo").unwrap();
}
133 changes: 93 additions & 40 deletions xtask/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::{
fmt::Write as _,
fs::{copy, create_dir_all, OpenOptions},
io::{BufRead as _, BufReader, Write as _},
path::PathBuf,
path::{Path, PathBuf},
process::{Child, ChildStdin, Command, Output, Stdio},
sync::{Arc, Mutex},
thread,
Expand Down Expand Up @@ -46,7 +46,7 @@ enum Environment {
///
/// find . -name '*.deb' -print0 \
/// | xargs -0 -I {} sh -c "dpkg --fsys-tarfile {} \
/// | tar --wildcards --extract '*vmlinuz*' --file -"
/// | tar --wildcards --extract '*vmlinuz*' '**/modules/*' --file -"
#[clap(required = true)]
kernel_image: Vec<PathBuf>,
},
Expand Down Expand Up @@ -298,21 +298,38 @@ pub fn run(opts: Options) -> Result<()> {

let target = format!("{guest_arch}-unknown-linux-musl");

// Build our init program. The contract is that it will run anything it finds in /bin.
let init = build(Some(&target), |cmd| {
cmd.args(["--package", "init", "--profile", "release"])
// Our mininal test linux distro contains:
// - 'init' which runs all binaries in /bin
// - 'modprobe' which is able to load kernel modules
let distro_binaries = build(Some(&target), |cmd| {
cmd.args(["--package", "test-distro", "--profile", "release"])
})
.context("building init program failed")?;

let init = match &*init {
[(name, init)] => {
if name != "init" {
bail!("expected init program to be named init, found {name}")
}
init
}
init => bail!("expected exactly one init program, found {init:?}"),
};
.context("building test-distro program failed")?;

let init = distro_binaries
.iter()
.find(|(name, _)| name == "init")
.map(|(_, path)| path)
.ok_or_else(|| anyhow!("init not found"))?;

let modprobe = distro_binaries
.iter()
.find(|(name, _)| name == "modprobe")
.map(|(_, path)| path)
.ok_or_else(|| anyhow!("modprobe not found"))?;

let version = kernel_image
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| anyhow!("kernel_image file_name failed"))?;
let version = version.trim_start_matches("vmlinuz-");
let modules_dir = kernel_image
.parent()
.expect("unable to get parent directory from kernel_image")
.join("../usr/lib/modules/")
.join(version)
.canonicalize()
.with_context(|| format!("failed to canonicalize {version}"))?;

let binaries = binaries(Some(&target))?;

Expand All @@ -337,23 +354,31 @@ pub fn run(opts: Options) -> Result<()> {
let Child { stdin, .. } = &mut gen_init_cpio_child;
let mut stdin = stdin.take().unwrap();

use std::os::unix::ffi::OsStrExt as _;

// Send input into gen_init_cpio which looks something like
//
// file /init path-to-init 0755 0 0
// dir /bin 0755 0 0
// file /bin/foo path-to-foo 0755 0 0
// file /bin/bar path-to-bar 0755 0 0

for bytes in [
"file /init ".as_bytes(),
init.as_os_str().as_bytes(),
" 0755 0 0\n".as_bytes(),
"dir /bin 0755 0 0\n".as_bytes(),
] {
stdin.write_all(bytes).expect("write");
}
// file /init path-to-init 755 0 0
// dir /bin 755 0 0
// file /bin/foo path-to-foo 755 0 0
// file /bin/bar path-to-bar 755 0 0

let mut input = vec![];

writeln!(input, "file /init {} 755 0 0", init.to_string_lossy()).expect("write");
writeln!(input, "dir /bin 755 0 0").expect("write");
writeln!(input, "dir /sbin 755 0 0").expect("write");
writeln!(input, "dir /usr 755 0 0").expect("write");
writeln!(input, "dir /usr/sbin 755 0 0").expect("write");
writeln!(input, "dir /usr/bin 755 0 0").expect("write");
writeln!(
input,
"file /sbin/modprobe {} 755 0 0",
modprobe.to_string_lossy()
)
.expect("write");
writeln!(input, "dir /lib 755 0 0").expect("write");
writeln!(input, "dir /lib/modules 755 0 0").expect("write");
cpioify_dir(&modules_dir, "/lib/modules", &mut input)
.context("cpioify_dir failed")?;

for (profile, binaries) in binaries {
for (name, binary) in binaries {
Expand All @@ -362,17 +387,17 @@ pub fn run(opts: Options) -> Result<()> {
copy(&binary, &path).with_context(|| {
format!("copy({}, {}) failed", binary.display(), path.display())
})?;
for bytes in [
"file /bin/".as_bytes(),
name.as_bytes(),
" ".as_bytes(),
path.as_os_str().as_bytes(),
" 0755 0 0\n".as_bytes(),
] {
stdin.write_all(bytes).expect("write");
}
writeln!(
input,
"file /bin/{} {} 755 0 0",
name,
path.to_string_lossy()
)
.expect("write");
}
}

stdin.write_all(&input).expect("write");
// Must explicitly close to signal EOF.
drop(stdin);

Expand Down Expand Up @@ -413,6 +438,7 @@ pub fn run(opts: Options) -> Result<()> {
//
// Heed the advice and boot with noapic. We don't know why this happens.
kernel_args.push(" noapic");
kernel_args.push(" init=/sbin/init");
qemu.args(["-no-reboot", "-nographic", "-m", "512M", "-smp", "2"])
.arg("-append")
.arg(kernel_args)
Expand Down Expand Up @@ -526,3 +552,30 @@ pub fn run(opts: Options) -> Result<()> {
}
}
}

fn cpioify_dir(dir: &Path, base: &str, input: &mut Vec<u8>) -> Result<()> {
for entry in std::fs::read_dir(dir).context("read_dir failed")? {
let entry = entry.context("read_dir failed")?;
let path = entry.path();
let metadata = entry.metadata().context("metadata failed")?;
let file_type = metadata.file_type();
let file_name = path
.file_name()
.ok_or_else(|| anyhow!("file_name failed"))?;
let file_name = file_name.to_str().ok_or_else(|| anyhow!("to_str failed"))?;
if file_type.is_dir() {
writeln!(input, "dir {}/{} 755 0 0", base, file_name).expect("write");
cpioify_dir(&path, format!("{}/{}", base, file_name).as_str(), input)?;
} else if file_type.is_file() {
writeln!(
input,
"file {}/{} {} 644 0 0",
base,
file_name,
path.to_string_lossy(),
)
.expect("write");
}
}
Ok(())
}

0 comments on commit bc64c61

Please sign in to comment.