Added benchmarks and fixed a parallelism bug

crates/resolved-cli/src/main.rs

@@ -109,6 +109,7 @@ fn render(
         },
         appearance: file.appearance(),
         read_radiance: false,
+        profile: timings.is_some(),
     };
 
     let start = Instant::now();
@@ -121,6 +122,7 @@ fn render(
     let loading = start.elapsed();
 
     let mut frame_times = Vec::new();
+    let mut pass_times = Vec::new();
     let mut frame = None;
     for _ in 0..timings.unwrap_or(1).max(1) {
         let start = Instant::now();
@@ -129,6 +131,7 @@ fn render(
             .and_then(|p| p.wait())
             .map_err(|e| e.to_string())?;
         frame_times.push(start.elapsed().as_secs_f64() * 1e3);
+        pass_times.extend(rendered.timings);
         frame = Some(rendered);
     }
     let frame = frame.expect("at least one frame");
@@ -153,6 +156,23 @@ fn render(
                 warm.len()
             );
         }
+        // GPU time per pass, median over the frames after the first.
+        if pass_times.len() > 1 {
+            let median = |f: fn(&resolved::gpu::profile::PassTimings) -> f64| {
+                let mut v: Vec<f64> = pass_times[1..].iter().map(f).collect();
+                v.sort_by(f64::total_cmp);
+                v[v.len() / 2]
+            };
+            println!(
+                "gpu      bodies {:.3} ms, mesh {:.3} ms, sensor {:.3} ms, first to last {:.3} ms (medians)",
+                median(|t| t.bodies),
+                median(|t| t.mesh),
+                median(|t| t.sensor),
+                median(|t| t.total)
+            );
+        } else if pass_times.is_empty() {
+            println!("gpu      no pass timings: this adapter has no timestamp queries");
+        }
     }
     let detector = &renderer.camera().detector;
     if let Some(out) = out {

crates/resolved/Cargo.toml

@@ -35,3 +35,8 @@ stl_io = "0.11"
 [dev-dependencies]
 approx = "0.5"
 image = { version = "0.25", default-features = false, features = ["png"] }
+criterion = "0.8"
+
+[[bench]]
+name = "render"
+harness = false

crates/resolved/benches/render.rs

@@ -0,0 +1,128 @@
+//! End-to-end frame times: `submit` then `wait`, including reading the DN image back.
+//!
+//! Every frame shows a model in the example scenes' composition: an isometric view
+//! from low orbit with Earth's limb 2 degrees above the image center. Models are the
+//! box-wing from the repository and, if `scripts/fetch-models.sh` has been run,
+//! Landsat 8 (85k triangles) and ICESat-2 (752k triangles). Each runs on the GPU and
+//! on lavapipe at 1024 x 1024 and 1920 x 1080.
+//!
+//! Per-pass GPU times are not here; `resolved render --timings N` reports them.
+
+use std::path::{Path, PathBuf};
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use resolved::gpu::AdapterChoice;
+use resolved::nalgebra::{Matrix3, Rotation3, UnitQuaternion, Vector3};
+use resolved::radiometry::AU;
+use resolved::{CameraConfig, LoadOptions, Mesh, Pose, Renderer, RendererOptions, Scene, Target};
+
+const EARTH_RADIUS: f64 = 6_378_137.0;
+const ORBIT: f64 = 7.0e6;
+
+fn examples() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/models")
+}
+
+/// The models to bench, skipping NASA models that have not been fetched.
+fn models() -> Vec<(&'static str, Mesh)> {
+    let mut models = vec![(
+        "boxwing",
+        Mesh::load(&examples().join("boxwing.gltf"), &LoadOptions::default()).unwrap(),
+    )];
+    for name in ["landsat8", "icesat2"] {
+        let path = examples().join("nasa").join(format!("{name}.glb"));
+        if path.is_file() {
+            models.push((name, Mesh::load(&path, &LoadOptions::default()).unwrap()));
+        } else {
+            eprintln!("skipping {name}: run scripts/fetch-models.sh to bench it");
+        }
+    }
+    models
+}
+
+/// The scene and observer for `mesh`, as in `scripts/make-example-scenes.py`.
+fn limb_view(mesh: &Mesh, model: resolved::ModelId) -> (Scene, Pose) {
+    let (mut lo, mut hi) = (Vector3::repeat(f64::MAX), Vector3::repeat(f64::MIN));
+    for p in &mesh.positions {
+        let p = Vector3::from(*p).cast::<f64>();
+        lo = lo.inf(&p);
+        hi = hi.sup(&p);
+    }
+    let center = (lo + hi) / 2.0;
+    let radius = (hi - lo).norm() / 2.0;
+
+    let limb = (EARTH_RADIUS / ORBIT).asin();
+    let theta = limb - 2f64.to_radians();
+    let look = Vector3::new(ORBIT, 0.0, 0.0);
+    let boresight = Vector3::new(-theta.cos(), theta.sin(), 0.0);
+    // Body (1, 1, 1) toward the camera, body +Z toward the local vertical.
+    let frame = |a: Vector3<f64>, b: Vector3<f64>| {
+        let t1 = a.normalize();
+        let t2 = a.cross(&b).normalize();
+        Matrix3::from_columns(&[t1, t2, t1.cross(&t2)])
+    };
+    let rotation =
+        frame(-boresight, Vector3::x()) * frame(Vector3::repeat(1.0), Vector3::z()).transpose();
+    let attitude = UnitQuaternion::from_rotation_matrix(&Rotation3::from_matrix(&rotation));
+    let distance = radius / 2.8f64.to_radians().tan();
+    let observer = Pose::looking_at(look - boresight * distance, look, Vector3::x()).unwrap();
+    let scene = Scene {
+        earth_attitude: UnitQuaternion::identity(),
+        sun: Vector3::new(0.743, -0.669, 0.017).normalize() * AU,
+        moon: Vector3::new(0.0, 0.0, -3.844e8),
+        targets: vec![Target {
+            model,
+            pose: Pose::new(look - attitude * center, attitude),
+        }],
+    };
+    (scene, observer)
+}
+
+fn camera(width: u32, height: u32) -> CameraConfig {
+    let mut camera = CameraConfig::example();
+    camera.width = width;
+    camera.height = height;
+    camera
+}
+
+fn frames(c: &mut Criterion) {
+    let models = models();
+    for (device, adapter) in [("gpu", AdapterChoice::Gpu), ("cpu", AdapterChoice::Cpu)] {
+        let mut group = c.benchmark_group(format!("frame/{device}"));
+        if device == "cpu" {
+            group.sample_size(20);
+        }
+        for (width, height) in [(1024, 1024), (1920, 1080)] {
+            let options = RendererOptions {
+                adapter: adapter.clone(),
+                ..Default::default()
+            };
+            let mut renderer = Renderer::new(camera(width, height), options).unwrap();
+            for (name, mesh) in &models {
+                let model = renderer.load_model(mesh).unwrap();
+                let (scene, observer) = limb_view(mesh, model);
+                let id = BenchmarkId::new(format!("{width}x{height}"), name);
+                group.bench_function(id, |b| {
+                    b.iter(|| renderer.submit(&scene, &observer).unwrap().wait().unwrap())
+                });
+            }
+        }
+        group.finish();
+    }
+}
+
+fn setup(c: &mut Criterion) {
+    let mut group = c.benchmark_group("setup");
+    group.sample_size(10);
+    // Create and drop as one cycle. Timing creation alone depends on when criterion
+    // drops the previous renderer, and with the driver unloading in between it came out
+    // bimodal (16 or 160 ms) before the Vulkan instance was shared.
+    group.bench_function("renderer/gpu/1920x1080", |b| {
+        b.iter(|| drop(Renderer::new(camera(1920, 1080), RendererOptions::default()).unwrap()))
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, frames, setup);
+criterion_main!(benches);

crates/resolved/src/gpu/bodies.rs

@@ -6,6 +6,7 @@
 
 use nalgebra::Vector3;
 
+use super::profile::PassTimestamps;
 use super::targets::{FrameTargets, DEPTH_COMPARE, DEPTH_FORMAT, RADIANCE_FORMAT};
 use super::GpuContext;
 use crate::bodies::{Appearance, EARTH_EQUATORIAL_RADIUS, EARTH_POLAR_RADIUS, MOON_RADIUS};
@@ -186,13 +187,15 @@ impl BodiesPass {
         }
     }
 
-    /// Uploads this frame's uniforms and records the pass into `encoder`.
+    /// Uploads this frame's uniforms and records the pass into `encoder`, writing
+    /// `timestamps` around it if given.
     pub fn encode(
         &self,
         ctx: &GpuContext,
         encoder: &mut wgpu::CommandEncoder,
         targets: &FrameTargets,
         uniforms: &BodiesUniforms,
+        timestamps: Option<PassTimestamps>,
     ) {
         ctx.queue
             .write_buffer(&self.uniforms, 0, bytemuck::bytes_of(uniforms));
@@ -217,6 +220,7 @@ impl BodiesPass {
                 }),
                 stencil_ops: None,
             }),
+            timestamp_writes: timestamps.map(PassTimestamps::render),
             ..Default::default()
         });
         pass.set_pipeline(&self.pipeline);

crates/resolved/src/gpu/context.rs

@@ -41,25 +41,32 @@ impl std::fmt::Debug for GpuContext {
 
 /// Every adapter this machine offers, in the order the drivers report them.
 pub fn adapters() -> Vec<wgpu::AdapterInfo> {
-    let instance = instance();
-    pollster::block_on(instance.enumerate_adapters(BACKENDS))
+    pollster::block_on(instance().enumerate_adapters(BACKENDS))
         .iter()
         .map(|a| a.get_info())
         .collect()
 }
 
-fn instance() -> wgpu::Instance {
-    wgpu::Instance::new(wgpu::InstanceDescriptor {
-        backends: BACKENDS,
-        ..wgpu::InstanceDescriptor::new_without_display_handle_from_env()
+/// The one wgpu instance this process uses, created on first use and never dropped.
+///
+/// One per renderer would be simpler, but creating and destroying Vulkan instances on
+/// several threads at once crashes the Vulkan loader (1.4.363 here): when the last
+/// instance goes, it unloads the drivers, and an instance being created on another
+/// thread at that moment calls into a driver that is no longer loaded.
+fn instance() -> &'static wgpu::Instance {
+    static INSTANCE: std::sync::OnceLock<wgpu::Instance> = std::sync::OnceLock::new();
+    INSTANCE.get_or_init(|| {
+        wgpu::Instance::new(wgpu::InstanceDescriptor {
+            backends: BACKENDS,
+            ..wgpu::InstanceDescriptor::new_without_display_handle_from_env()
+        })
     })
 }
 
 impl GpuContext {
     /// Opens a device on the adapter `choice` selects.
     pub fn new(choice: &AdapterChoice) -> Result<Self, Error> {
-        let instance = instance();
-        let candidates = pollster::block_on(instance.enumerate_adapters(BACKENDS));
+        let candidates = pollster::block_on(instance().enumerate_adapters(BACKENDS));
         let infos: Vec<_> = candidates.iter().map(|a| a.get_info()).collect();
         let kinds: Vec<_> = infos
             .iter()

crates/resolved/src/gpu/mesh.rs

@@ -7,6 +7,7 @@
 use nalgebra::{Matrix3, Vector3};
 
 use super::bodies::MAX_CHANNELS;
+use super::profile::PassTimestamps;
 use super::targets::{FrameTargets, DEPTH_COMPARE, DEPTH_FORMAT, RADIANCE_FORMAT};
 use super::GpuContext;
 use crate::camera::Intrinsics;
@@ -284,6 +285,7 @@ impl MeshPass {
         encoder: &mut wgpu::CommandEncoder,
         targets: &FrameTargets,
         draws: &[(&GpuMesh, MeshDraw)],
+        timestamps: Option<PassTimestamps>,
     ) {
         if draws.len() > self.capacity {
             self.capacity = draws.len().next_power_of_two();
@@ -318,6 +320,7 @@ impl MeshPass {
                 }),
                 stencil_ops: None,
             }),
+            timestamp_writes: timestamps.map(PassTimestamps::render),
             ..Default::default()
         });
         pass.set_pipeline(&self.pipeline);

crates/resolved/src/gpu/mod.rs

@@ -3,6 +3,7 @@
 pub mod bodies;
 pub mod context;
 pub mod mesh;
+pub mod profile;
 pub mod sensor;
 pub mod targets;
 

crates/resolved/src/gpu/profile.rs

@@ -0,0 +1,127 @@
+//! GPU timestamps around each pass, for seeing where a frame's time goes.
+//!
+//! Each pass writes a timestamp when it starts and when it ends. After the frame the
+//! queries are resolved into a buffer and read back with the image. Needs the
+//! adapter's `TIMESTAMP_QUERY` feature; without it there is nothing to measure and
+//! [`Profiler::new`] returns `None`.
+
+use super::GpuContext;
+
+/// The passes timed, in the order they run.
+pub const PASSES: [&str; 3] = ["bodies", "mesh", "sensor"];
+
+/// Where one pass writes its begin and end timestamps.
+#[derive(Clone, Copy, Debug)]
+pub struct PassTimestamps<'a> {
+    set: &'a wgpu::QuerySet,
+    begin: u32,
+}
+
+impl<'a> PassTimestamps<'a> {
+    /// For a render pass descriptor.
+    pub fn render(self) -> wgpu::RenderPassTimestampWrites<'a> {
+        wgpu::RenderPassTimestampWrites {
+            query_set: self.set,
+            beginning_of_pass_write_index: Some(self.begin),
+            end_of_pass_write_index: Some(self.begin + 1),
+        }
+    }
+
+    /// For a compute pass descriptor.
+    pub fn compute(self) -> wgpu::ComputePassTimestampWrites<'a> {
+        wgpu::ComputePassTimestampWrites {
+            query_set: self.set,
+            beginning_of_pass_write_index: Some(self.begin),
+            end_of_pass_write_index: Some(self.begin + 1),
+        }
+    }
+}
+
+/// GPU time spent in each pass of one frame, in milliseconds.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct PassTimings {
+    /// Earth, Moon and Sun.
+    pub bodies: f64,
+    /// Target meshes.
+    pub mesh: f64,
+    /// Radiance to DN.
+    pub sensor: f64,
+    /// From the start of the first pass to the end of the last, gaps included.
+    pub total: f64,
+}
+
+/// A timestamp query set with the buffers to read it back.
+#[derive(Debug)]
+pub struct Profiler {
+    set: wgpu::QuerySet,
+    resolve: wgpu::Buffer,
+    /// Read back with the frame; the renderer maps it alongside the image.
+    pub staging: wgpu::Buffer,
+    /// Nanoseconds per timestamp tick.
+    period: f64,
+}
+
+const QUERIES: u32 = 2 * PASSES.len() as u32;
+
+impl Profiler {
+    /// A profiler, if the device was opened with timestamp queries.
+    pub fn new(ctx: &GpuContext) -> Option<Self> {
+        if !ctx
+            .device
+            .features()
+            .contains(wgpu::Features::TIMESTAMP_QUERY)
+        {
+            return None;
+        }
+        let size = QUERIES as u64 * 8;
+        let buffer = |label, usage| {
+            ctx.device.create_buffer(&wgpu::BufferDescriptor {
+                label: Some(label),
+                size,
+                usage,
+                mapped_at_creation: false,
+            })
+        };
+        Some(Self {
+            set: ctx.device.create_query_set(&wgpu::QuerySetDescriptor {
+                label: Some("pass timestamps"),
+                ty: wgpu::QueryType::Timestamp,
+                count: QUERIES,
+            }),
+            resolve: buffer(
+                "timestamp resolve",
+                wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC,
+            ),
+            staging: buffer(
+                "timestamp staging",
+                wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
+            ),
+            period: ctx.queue.get_timestamp_period() as f64,
+        })
+    }
+
+    /// The timestamps for pass `index` of [`PASSES`].
+    pub fn pass(&self, index: usize) -> PassTimestamps<'_> {
+        PassTimestamps {
+            set: &self.set,
+            begin: 2 * index as u32,
+        }
+    }
+
+    /// Records resolving the queries into the staging buffer, after the last pass.
+    pub fn resolve(&self, encoder: &mut wgpu::CommandEncoder) {
+        encoder.resolve_query_set(&self.set, 0..QUERIES, &self.resolve, 0);
+        encoder.copy_buffer_to_buffer(&self.resolve, 0, &self.staging, 0, self.resolve.size());
+    }
+
+    /// Converts mapped staging contents to milliseconds.
+    pub fn timings(&self, ticks: &[u64]) -> PassTimings {
+        let ms = |a: u64, b: u64| b.wrapping_sub(a) as f64 * self.period * 1e-6;
+        PassTimings {
+            bodies: ms(ticks[0], ticks[1]),
+            mesh: ms(ticks[2], ticks[3]),
+            sensor: ms(ticks[4], ticks[5]),
+            total: ms(ticks[0], ticks[5]),
+        }
+    }
+}

crates/resolved/src/gpu/sensor.rs

@@ -4,6 +4,7 @@
 //! writes [`FrameTargets::dn`]. No noise yet (M3); the result for each pixel is
 //! [`Detector::dn`](crate::camera::Detector::dn) of [`Radiometry::electrons`].
 
+use super::profile::PassTimestamps;
 use super::targets::FrameTargets;
 use super::GpuContext;
 use crate::camera::CameraConfig;
@@ -88,6 +89,7 @@ impl SensorPass {
         encoder: &mut wgpu::CommandEncoder,
         targets: &FrameTargets,
         uniforms: &SensorUniforms,
+        timestamps: Option<PassTimestamps>,
     ) {
         ctx.queue
             .write_buffer(&self.uniforms, 0, bytemuck::bytes_of(uniforms));
@@ -115,7 +117,7 @@ impl SensorPass {
         let (x, y) = (groups.min(65_535), groups.div_ceil(65_535));
         let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
             label: Some("sensor"),
-            timestamp_writes: None,
+            timestamp_writes: timestamps.map(PassTimestamps::compute),
         });
         pass.set_pipeline(&self.pipeline);
         pass.set_bind_group(0, &bind_group, &[]);

crates/resolved/src/render.rs

@@ -15,6 +15,7 @@ use crate::camera::CameraConfig;
 use crate::eclipse::sun_visible_fraction;
 use crate::gpu::bodies::{BodiesPass, BodiesUniforms, BodyRadiometry};
 use crate::gpu::mesh::{GpuMesh, MeshDraw, MeshPass};
+use crate::gpu::profile::{PassTimings, Profiler};
 use crate::gpu::sensor::{SensorPass, SensorUniforms};
 use crate::gpu::targets::FrameTargets;
 use crate::gpu::{AdapterChoice, GpuContext};
@@ -33,6 +34,9 @@ pub struct RendererOptions {
     /// Also read back the channel radiance image each frame. Costs another 16 bytes per
     /// pixel of transfer; for debugging and tests.
     pub read_radiance: bool,
+    /// Time each pass on the GPU and report it in [`Frame::timings`]. Needs an adapter
+    /// with timestamp queries; without one, timings are `None`.
+    pub profile: bool,
 }
 
 /// One rendered image.
@@ -48,6 +52,9 @@ pub struct Frame {
     pub dn: Vec<u16>,
     /// Channel radiance, if [`RendererOptions::read_radiance`] was set.
     pub radiance: Option<Vec<[f32; 4]>>,
+    /// GPU time per pass, if [`RendererOptions::profile`] was set and the adapter
+    /// supports timestamps.
+    pub timings: Option<PassTimings>,
 }
 
 impl Frame {
@@ -58,6 +65,11 @@ impl Frame {
 }
 
 /// Renders frames for one camera.
+///
+/// Create and drop renderers from one thread at a time. With current drivers
+/// (NVIDIA 615.71 and Mesa's lavapipe, measured on mainframe), creating or dropping
+/// Vulkan devices on several threads at once occasionally crashes or hangs inside the
+/// driver. Rendering on several renderers at once, once they exist, is not affected.
 #[derive(Debug)]
 pub struct Renderer {
     ctx: GpuContext,
@@ -72,6 +84,7 @@ pub struct Renderer {
     models: Vec<GpuMesh>,
     dn_staging: wgpu::Buffer,
     radiance_staging: Option<(wgpu::Buffer, u32)>,
+    profiler: Option<Profiler>,
 }
 
 impl Renderer {
@@ -104,6 +117,11 @@ impl Renderer {
                 row,
             )
         });
+        let profiler = if options.profile {
+            Profiler::new(&ctx)
+        } else {
+            None
+        };
         Ok(Self {
             bodies: BodiesPass::new(&ctx),
             meshes: MeshPass::new(&ctx),
@@ -117,6 +135,7 @@ impl Renderer {
             models: Vec::new(),
             dn_staging,
             radiance_staging,
+            profiler,
         })
     }
 
@@ -176,11 +195,21 @@ impl Renderer {
 
         let ctx = &self.ctx;
         let mut encoder = ctx.device.create_command_encoder(&Default::default());
+        let stamps = |i| self.profiler.as_ref().map(|p| p.pass(i));
         self.bodies
-            .encode(ctx, &mut encoder, &self.targets, &bodies);
-        self.meshes.encode(ctx, &mut encoder, &self.targets, &draws);
-        self.sensor
-            .encode(ctx, &mut encoder, &self.targets, &self.sensor_uniforms);
+            .encode(ctx, &mut encoder, &self.targets, &bodies, stamps(0));
+        self.meshes
+            .encode(ctx, &mut encoder, &self.targets, &draws, stamps(1));
+        self.sensor.encode(
+            ctx,
+            &mut encoder,
+            &self.targets,
+            &self.sensor_uniforms,
+            stamps(2),
+        );
+        if let Some(profiler) = &self.profiler {
+            profiler.resolve(&mut encoder);
+        }
         encoder.copy_buffer_to_buffer(
             &self.targets.dn,
             0,
@@ -211,9 +240,7 @@ impl Renderer {
 
         // Ask for the staging buffers now; the mapping completes when the GPU finishes.
         let (sender, mapped) = mpsc::channel();
-        let staged =
-            std::iter::once(&self.dn_staging).chain(self.radiance_staging.as_ref().map(|(b, _)| b));
-        for buffer in staged {
+        for buffer in self.staging_buffers() {
             let sender = sender.clone();
             buffer
                 .slice(..)
@@ -225,6 +252,13 @@ impl Renderer {
             mapped,
         })
     }
+
+    /// Every buffer read back after a frame.
+    fn staging_buffers(&self) -> impl Iterator<Item = &wgpu::Buffer> {
+        std::iter::once(&self.dn_staging)
+            .chain(self.radiance_staging.as_ref().map(|(b, _)| b))
+            .chain(self.profiler.as_ref().map(|p| &p.staging))
+    }
 }
 
 /// A frame the GPU is working on. [`Pending::wait`] returns it.
@@ -271,12 +305,28 @@ impl Pending<'_> {
                 Some(texels)
             }
         };
+        let timings = match &r.profiler {
+            None => None,
+            Some(profiler) => {
+                let timings = {
+                    let view = profiler
+                        .staging
+                        .slice(..)
+                        .get_mapped_range()
+                        .map_err(gpu_error)?;
+                    profiler.timings(bytemuck::cast_slice::<u8, u64>(&view))
+                };
+                profiler.staging.unmap();
+                Some(timings)
+            }
+        };
         Ok(Frame {
             width: r.camera.width,
             height: r.camera.height,
             channels: r.camera.channels.len() as u32,
             dn,
             radiance,
+            timings,
         })
     }
 
@@ -293,8 +343,7 @@ impl Pending<'_> {
                 timeout: None,
             })
             .map_err(gpu_error)?;
-        let buffers = 1 + usize::from(r.radiance_staging.is_some());
-        for _ in 0..buffers {
+        for _ in r.staging_buffers() {
             self.mapped.recv().map_err(gpu_error)?.map_err(gpu_error)?;
         }
         Ok(())
@@ -304,8 +353,7 @@ impl Pending<'_> {
 impl Drop for Pending<'_> {
     fn drop(&mut self) {
         if self.index.is_some() && self.block().is_ok() {
-            self.renderer.dn_staging.unmap();
-            if let Some((buffer, _)) = &self.renderer.radiance_staging {
+            for buffer in self.renderer.staging_buffers() {
                 buffer.unmap();
             }
         }

crates/resolved/tests/bodies.rs

@@ -60,7 +60,7 @@ fn render_with(
         &body_radiometry(),
     );
     let mut encoder = ctx.device.create_command_encoder(&Default::default());
-    pass.encode(ctx, &mut encoder, &targets, &uniforms);
+    pass.encode(ctx, &mut encoder, &targets, &uniforms, None);
     ctx.queue.submit([encoder.finish()]);
     let radiance = targets.read_radiance(ctx);
     common::save(name, &ctx.info, &radiance);

crates/resolved/tests/common/mod.rs

@@ -15,6 +15,16 @@ use resolved::radiometry::{Radiometry, AU};
 use resolved::CameraConfig;
 use wgpu::AdapterInfo;
 
+/// Held by any test that creates or drops a renderer, so only one does at a time.
+///
+/// Creating and dropping Vulkan devices on several threads at once crashes or hangs
+/// the drivers on mainframe (NVIDIA 615.71, lavapipe): measured 4 crashes and 2 hangs
+/// in 150 runs of the renderer tests in parallel, none in 100 run one at a time.
+pub fn device_lock() -> std::sync::MutexGuard<'static, ()> {
+    static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+    LOCK.lock().unwrap_or_else(|e| e.into_inner())
+}
+
 /// A hardware GPU and lavapipe, opened once per test binary and shared by its tests.
 pub fn contexts() -> &'static [GpuContext; 2] {
     static CONTEXTS: OnceLock<[GpuContext; 2]> = OnceLock::new();

crates/resolved/tests/gpu_context.rs

@@ -3,6 +3,8 @@
 //! Needs a Vulkan GPU and Mesa's lavapipe (`vulkan-swrast` on Arch,
 //! `mesa-vulkan-drivers` on Debian and Ubuntu).
 
+mod common;
+
 use resolved::gpu::{AdapterChoice, GpuContext};
 use wgpu::util::DeviceExt;
 
@@ -72,6 +74,7 @@ fn double_plus_one(ctx: &GpuContext, input: &[f32]) -> Vec<f32> {
 }
 
 fn check(choice: AdapterChoice, expect_cpu: bool) {
+    let _devices = common::device_lock();
     let ctx = GpuContext::new(&choice).unwrap();
     assert_eq!(ctx.is_cpu(), expect_cpu, "{choice:?} picked {:?}", ctx.info);
     let input: Vec<f32> = (0..1000).map(|i| i as f32 * 0.5).collect();

crates/resolved/tests/mesh.rs

@@ -117,8 +117,8 @@ fn render(ctx: &GpuContext, name: &str, scene: &Scene, camera: &Pose, mesh: &Mes
         })
         .collect();
     let mut encoder = ctx.device.create_command_encoder(&Default::default());
-    bodies.encode(ctx, &mut encoder, &targets, &uniforms);
-    meshes.encode(ctx, &mut encoder, &targets, &draws);
+    bodies.encode(ctx, &mut encoder, &targets, &uniforms, None);
+    meshes.encode(ctx, &mut encoder, &targets, &draws, None);
     ctx.queue.submit([encoder.finish()]);
     let radiance = targets.read_radiance(ctx);
     common::save(name, &ctx.info, &radiance);

crates/resolved/tests/renderer.rs

@@ -80,6 +80,7 @@ fn cube_target(model: ModelId) -> Target {
 
 #[test]
 fn sunlit_cube_matches_the_cpu_chain() {
+    let _devices = common::device_lock();
     let config = CameraConfig::example();
     let r = Radiometry::new(&config);
     let ray = config.intrinsics().ray(&Vector2::new(960.5, 540.5));
@@ -108,6 +109,7 @@ fn sunlit_cube_matches_the_cpu_chain() {
 
 #[test]
 fn camera_mount_is_rigid() {
+    let _devices = common::device_lock();
     // Mounting the camera rotated on the observer, and turning the observer the other
     // way, must give the same image as no mount at all.
     let turn = UnitQuaternion::from_euler_angles(0.3, -0.2, 1.1);
@@ -155,6 +157,7 @@ fn camera_mount_is_rigid() {
 
 #[test]
 fn frames_are_independent() {
+    let _devices = common::device_lock();
     for mut renderer in renderers(false) {
         let model = renderer.load_model(&cube()).unwrap();
         let with = scene(vec![cube_target(model)]);
@@ -185,8 +188,44 @@ fn frames_are_independent() {
 
 #[test]
 fn unknown_model_is_an_error() {
+    let _devices = common::device_lock();
     let mut renderer = Renderer::new(CameraConfig::example(), Default::default()).unwrap();
     assert!(renderer
         .submit(&scene(vec![cube_target(ModelId(0))]), &observer())
         .is_err());
 }
+
+#[test]
+fn profiling_reports_pass_times() {
+    let _devices = common::device_lock();
+    for profile in [false, true] {
+        for adapter in [AdapterChoice::Gpu, AdapterChoice::Cpu] {
+            let options = RendererOptions {
+                adapter,
+                profile,
+                ..Default::default()
+            };
+            let mut renderer = Renderer::new(CameraConfig::example(), options).unwrap();
+            let model = renderer.load_model(&cube()).unwrap();
+            let frame = renderer
+                .submit(&scene(vec![cube_target(model)]), &observer())
+                .unwrap()
+                .wait()
+                .unwrap();
+            let name = &renderer.adapter_info().name;
+            match (profile, frame.timings) {
+                (false, timings) => assert!(timings.is_none(), "{name}"),
+                (true, None) => {
+                    panic!("{name}: no timings; both mainframe adapters support timestamps")
+                }
+                (true, Some(t)) => {
+                    for pass in [t.bodies, t.mesh, t.sensor] {
+                        assert!(pass > 0.0 && pass <= t.total, "{name}: {t:?}");
+                    }
+                    // A frame takes under a second even on lavapipe.
+                    assert!(t.total < 1000.0, "{name}: {t:?}");
+                }
+            }
+        }
+    }
+}

crates/resolved/tests/sensor.rs

@@ -35,7 +35,7 @@ fn sense(ctx: &GpuContext, name: &str, radiance: &[[f32; 4]]) -> Vec<u16> {
     let pass = SensorPass::new(ctx);
     let uniforms = SensorUniforms::new(&config, &Radiometry::new(&config));
     let mut encoder = ctx.device.create_command_encoder(&Default::default());
-    pass.encode(ctx, &mut encoder, &targets, &uniforms);
+    pass.encode(ctx, &mut encoder, &targets, &uniforms, None);
     ctx.queue.submit([encoder.finish()]);
     let dn = targets.read_dn(ctx);
     common::save_dn(name, &ctx.info, &dn);
@@ -160,8 +160,8 @@ fn sunlit_earth_through_the_whole_chain() {
     for ctx in common::contexts() {
         let targets = FrameTargets::new(ctx, config.width, config.height, 3);
         let mut encoder = ctx.device.create_command_encoder(&Default::default());
-        BodiesPass::new(ctx).encode(ctx, &mut encoder, &targets, &bodies);
-        SensorPass::new(ctx).encode(ctx, &mut encoder, &targets, &sensor);
+        BodiesPass::new(ctx).encode(ctx, &mut encoder, &targets, &bodies, None);
+        SensorPass::new(ctx).encode(ctx, &mut encoder, &targets, &sensor, None);
         ctx.queue.submit([encoder.finish()]);
         let dn = targets.read_dn(ctx);
         common::save_dn("sensor-earth", &ctx.info, &dn);