Line data Source code
1 : //! This crate provides convenient access to the [`compiletest_rs`] package for testing [Dylint]
2 : //! libraries.
3 : //!
4 : //! **Note: If your test has dependencies, you must use `ui_test_example` or `ui_test_examples`.**
5 : //! See the [`question_mark_in_expression`] example in this repository.
6 : //!
7 : //! This crate provides the following three functions:
8 : //!
9 : //! - [`ui_test`] - test a library on all source files in a directory
10 : //! - [`ui_test_example`] - test a library on one example target
11 : //! - [`ui_test_examples`] - test a library on all example targets
12 : //!
13 : //! For most situations, you can add the following to your library's `lib.rs` file:
14 : //!
15 : //! ```rust,ignore
16 : //! #[test]
17 : //! fn ui() {
18 : //! dylint_testing::ui_test(env!("CARGO_PKG_NAME"), "ui");
19 : //! }
20 : //! ```
21 : //!
22 : //! And include one or more `.rs` and `.stderr` files in a `ui` directory alongside your library's
23 : //! `src` directory. See the [examples] in this repository.
24 : //!
25 : //! # Test builder
26 : //!
27 : //! In addition to the above three functions, [`ui::Test`] is a test "builder." Currently, the main
28 : //! advantage of using `Test` over the above functions is that `Test` allows flags to be passed to
29 : //! `rustc`. For an example of its use, see [`non_thread_safe_call_in_test`] in this repository.
30 : //!
31 : //! `Test` has three constructors, which correspond to the above three functions as follows:
32 : //!
33 : //! - [`ui::Test::src_base`] <-> [`ui_test`]
34 : //! - [`ui::Test::example`] <-> [`ui_test_example`]
35 : //! - [`ui::Test::examples`] <-> [`ui_test_examples`]
36 : //!
37 : //! In each case, the constructor's arguments are exactly those of the corresponding function.
38 : //!
39 : //! A `Test` instance has the following methods:
40 : //!
41 : //! - `dylint_toml` - set the `dylint.toml` file's contents (for testing [configurable libraries])
42 : //! - `rustc_flags` - pass flags to the compiler when running the test
43 : //! - `run` - run the test
44 : //!
45 : //! # Updating `.stderr` files
46 : //!
47 : //! If the standard error that results from running your `.rs` file differs from the contents of
48 : //! your `.stderr` file, `compiletest_rs` will produce a report like the following:
49 : //!
50 : //! ```text
51 : //! diff of stderr:
52 : //!
53 : //! error: calling `std::env::set_var` in a test could affect the outcome of other tests
54 : //! --> $DIR/main.rs:8:5
55 : //! |
56 : //! LL | std::env::set_var("KEY", "VALUE");
57 : //! | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
58 : //! |
59 : //! = note: `-D non-thread-safe-call-in-test` implied by `-D warnings`
60 : //!
61 : //! -error: aborting due to previous error
62 : //! +error: calling `std::env::set_var` in a test could affect the outcome of other tests
63 : //! + --> $DIR/main.rs:23:9
64 : //! + |
65 : //! +LL | std::env::set_var("KEY", "VALUE");
66 : //! + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
67 : //! +
68 : //! +error: aborting due to 2 previous errors
69 : //!
70 : //!
71 : //!
72 : //! The actual stderr differed from the expected stderr.
73 : //! Actual stderr saved to ...
74 : //! ```
75 : //!
76 : //! The meaning of each line is as follows:
77 : //!
78 : //! - A line beginning with a plus (`+`) is in the actual standard error, but not in your `.stderr`
79 : //! file.
80 : //! - A line beginning with a minus (`-`) is in your `.stderr` file, but not in the actual standard
81 : //! error.
82 : //! - A line beginning with a space (` `) is in both the actual standard error and your `.stderr`
83 : //! file, and is provided for context.
84 : //! - All other lines (e.g., `diff of stderr:`) contain `compiletest_rs` messages.
85 : //!
86 : //! **Note:** In the actual standard error, a blank line usually follows the `error: aborting due to
87 : //! N previous errors` line. So a correct `.stderr` file will typically contain one blank line at
88 : //! the end.
89 : //!
90 : //! In general, it is not too hard to update a `.stderr` file by hand. However, the `compiletest_rs`
91 : //! report should contain a line of the form `Actual stderr saved to PATH`. Copying `PATH` to your
92 : //! `.stderr` file should update it completely.
93 : //!
94 : //! Additional documentation on `compiletest_rs` can be found in [its repository].
95 : //!
96 : //! [Dylint]: https://github.com/trailofbits/dylint/tree/master
97 : //! [`compiletest_rs`]: https://github.com/Manishearth/compiletest-rs
98 : //! [`non_thread_safe_call_in_test`]: https://github.com/trailofbits/dylint/tree/master/examples/general/non_thread_safe_call_in_test/src/lib.rs
99 : //! [`question_mark_in_expression`]: https://github.com/trailofbits/dylint/tree/master/examples/restriction/question_mark_in_expression/Cargo.toml
100 : //! [`ui::Test::example`]: https://docs.rs/dylint_testing/latest/dylint_testing/ui/struct.Test.html#method.example
101 : //! [`ui::Test::examples`]: https://docs.rs/dylint_testing/latest/dylint_testing/ui/struct.Test.html#method.examples
102 : //! [`ui::Test::src_base`]: https://docs.rs/dylint_testing/latest/dylint_testing/ui/struct.Test.html#method.src_base
103 : //! [`ui::Test`]: https://docs.rs/dylint_testing/latest/dylint_testing/ui/struct.Test.html
104 : //! [`ui_test_example`]: https://docs.rs/dylint_testing/latest/dylint_testing/fn.ui_test_example.html
105 : //! [`ui_test_examples`]: https://docs.rs/dylint_testing/latest/dylint_testing/fn.ui_test_examples.html
106 : //! [`ui_test`]: https://docs.rs/dylint_testing/latest/dylint_testing/fn.ui_test.html
107 : //! [configurable libraries]: https://github.com/trailofbits/dylint/tree/master#configurable-libraries
108 : //! [docs.rs documentation]: https://docs.rs/dylint_testing/latest/dylint_testing/
109 : //! [examples]: https://github.com/trailofbits/dylint/tree/master/examples
110 : //! [its repository]: https://github.com/Manishearth/compiletest-rs
111 :
112 : use anyhow::{anyhow, ensure, Context, Result};
113 : use cargo_metadata::{Metadata, Package, Target};
114 : use compiletest_rs as compiletest;
115 : use dylint_internal::{env, library_filename, rustup::is_rustc, CommandExt};
116 : use once_cell::sync::{Lazy, OnceCell};
117 : use regex::Regex;
118 : use std::{
119 : env::{consts, remove_var, set_var, var_os},
120 : ffi::{OsStr, OsString},
121 : fs::{copy, read_dir, remove_file},
122 : io::BufRead,
123 : path::{Path, PathBuf},
124 : sync::Mutex,
125 : };
126 :
127 : pub mod ui;
128 :
129 : static DRIVER: OnceCell<PathBuf> = OnceCell::new();
130 : static LINKING_FLAGS: OnceCell<Vec<String>> = OnceCell::new();
131 :
132 : /// Test a library on all source files in a directory.
133 : ///
134 : /// - `name` is the name of a Dylint library to be tested. (Often, this is the same as the package
135 : /// name.)
136 : /// - `src_base` is a directory containing:
137 : /// - source files on which to test the library (`.rs` files), and
138 : /// - the output those files should produce (`.stderr` files).
139 0 : pub fn ui_test(name: &str, src_base: impl AsRef<Path>) {
140 0 : ui::Test::src_base(name, src_base).run();
141 0 : }
142 :
143 : /// Test a library on one example target.
144 : ///
145 : /// - `name` is the name of a Dylint library to be tested.
146 : /// - `example` is an example target on which to test the library.
147 0 : pub fn ui_test_example(name: &str, example: &str) {
148 0 : ui::Test::example(name, example).run();
149 0 : }
150 :
151 : /// Test a library on all example targets.
152 : ///
153 : /// - `name` is the name of a Dylint library to be tested.
154 0 : pub fn ui_test_examples(name: &str) {
155 0 : ui::Test::examples(name).run();
156 0 : }
157 :
158 0 : fn initialize(name: &str) -> Result<&Path> {
159 0 : DRIVER
160 0 : .get_or_try_init(|| {
161 0 : let _ = env_logger::try_init();
162 0 :
163 0 : // smoelius: Try to order failures by how informative they are: failure to build the
164 0 : // library, failure to find the library, failure to build/find the driver.
165 0 :
166 0 : dylint_internal::cargo::build(&format!("library `{name}`"))
167 0 : .build()
168 0 : .success()?;
169 :
170 : // smoelius: `DYLINT_LIBRARY_PATH` must be set before `dylint_libs` is called.
171 : // smoelius: This was true when `dylint_libs` called `name_toolchain_map`, but that is
172 : // no longer the case. I am leaving the comment here for now in case removal
173 : // of the `name_toolchain_map` call causes a regression.
174 0 : let metadata = dylint_internal::cargo::current_metadata().unwrap();
175 0 : let dylint_library_path = metadata.target_directory.join("debug");
176 0 : set_var(env::DYLINT_LIBRARY_PATH, dylint_library_path);
177 :
178 0 : let dylint_libs = dylint_libs(name)?;
179 0 : let driver = dylint::driver_builder::get(
180 0 : &dylint::opts::Dylint::default(),
181 0 : env!("RUSTUP_TOOLCHAIN"),
182 0 : )?;
183 :
184 0 : set_var(env::CLIPPY_DISABLE_DOCS_LINKS, "true");
185 0 : set_var(env::DYLINT_LIBS, dylint_libs);
186 0 :
187 0 : Ok(driver)
188 0 : })
189 0 : .map(PathBuf::as_path)
190 0 : }
191 :
192 : #[doc(hidden)]
193 0 : pub fn dylint_libs(name: &str) -> Result<String> {
194 0 : let metadata = dylint_internal::cargo::current_metadata().unwrap();
195 0 : let rustup_toolchain = env::var(env::RUSTUP_TOOLCHAIN)?;
196 0 : let filename = library_filename(name, &rustup_toolchain);
197 0 : let path = metadata.target_directory.join("debug").join(filename);
198 0 : let paths = vec![path];
199 0 : serde_json::to_string(&paths).map_err(Into::into)
200 0 : }
201 :
202 0 : fn example_target(package: &Package, example: &str) -> Result<Target> {
203 0 : package
204 0 : .targets
205 0 : .iter()
206 0 : .find(|target| target.kind == ["example"] && target.name == example)
207 0 : .cloned()
208 0 : .ok_or_else(|| anyhow!("Could not find example `{}`", example))
209 0 : }
210 :
211 : #[allow(clippy::unnecessary_wraps)]
212 0 : fn example_targets(package: &Package) -> Result<Vec<Target>> {
213 0 : Ok(package
214 0 : .targets
215 0 : .iter()
216 0 : .filter(|target| target.kind == ["example"])
217 0 : .cloned()
218 0 : .collect())
219 0 : }
220 :
221 0 : fn run_example_test(
222 0 : driver: &Path,
223 0 : metadata: &Metadata,
224 0 : package: &Package,
225 0 : target: &Target,
226 0 : config: &ui::Config,
227 0 : ) -> Result<()> {
228 0 : let linking_flags = linking_flags(metadata, package, target)?;
229 0 : let file_name = target
230 0 : .src_path
231 0 : .file_name()
232 0 : .ok_or_else(|| anyhow!("Could not get file name"))?;
233 :
234 0 : let tempdir = tempfile::tempdir().with_context(|| "`tempdir` failed")?;
235 0 : let src_base = tempdir.path();
236 0 : let to = src_base.join(file_name);
237 0 :
238 0 : copy(&target.src_path, &to).with_context(|| {
239 0 : format!(
240 0 : "Could not copy `{}` to `{}`",
241 0 : target.src_path,
242 0 : to.to_string_lossy()
243 0 : )
244 0 : })?;
245 0 : ["fixed", "stderr", "stdout"]
246 0 : .map(|extension| copy_with_extension(&target.src_path, &to, extension).unwrap_or_default());
247 0 :
248 0 : let mut config = config.clone();
249 0 : config.rustc_flags.extend(linking_flags.iter().cloned());
250 0 :
251 0 : run_tests(driver, src_base, &config);
252 0 :
253 0 : Ok(())
254 0 : }
255 :
256 0 : fn linking_flags(
257 0 : metadata: &Metadata,
258 0 : package: &Package,
259 0 : target: &Target,
260 0 : ) -> Result<&'static [String]> {
261 0 : LINKING_FLAGS
262 0 : .get_or_try_init(|| {
263 0 : let rustc_flags = rustc_flags(metadata, package, target)?;
264 :
265 0 : let mut linking_flags = Vec::new();
266 0 :
267 0 : let mut iter = rustc_flags.into_iter();
268 0 : while let Some(flag) = iter.next() {
269 0 : if flag.starts_with("--edition=") {
270 0 : linking_flags.push(flag);
271 0 : } else if flag == "--extern" || flag == "-L" {
272 0 : let arg = next(&flag, &mut iter)?;
273 0 : linking_flags.extend([flag, arg.trim_matches('\'').to_owned()]);
274 0 : }
275 : }
276 :
277 0 : Ok(linking_flags)
278 0 : })
279 0 : .map(Vec::as_slice)
280 0 : }
281 :
282 : // smoelius: We need to recover the `rustc` flags used to build a target. I can see four options:
283 : //
284 : // * Use `cargo build --build-plan`
285 : // - Pros: Easily parsable JSON output
286 : // - Cons: Unstable and likely to be removed: https://github.com/rust-lang/cargo/issues/7614
287 : // * Parse the output of `cargo build --verbose`
288 : // - Pros: ?
289 : // - Cons: Not as easily parsable, requires synchronization (see below)
290 : // * Use a custom executor like Siderophile does: https://github.com/trailofbits/siderophile/blob/26c067306f6c2f66d9530dacef6b17dbf59cdf8c/src/trawl_source/mod.rs#L399
291 : // - Pros: Ground truth
292 : // - Cons: Seems a bit of a heavy lift (Note: I think Siderophile's approach was inspired by
293 : // `cargo-geiger`.)
294 : // * Set `RUSTC_WORKSPACE_WRAPPER` to something that logs `rustc` invocations
295 : // - Pros: Ground truth
296 : // - Cons: Requires a separate executable/script, portability could be an issue
297 : //
298 : // I am going with the second option for now, because it seems to be the least of all evils. This
299 : // decision may need to be revisited.
300 :
301 0 : static RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^\s*Running\s*`(.*)`$").unwrap());
302 :
303 0 : fn rustc_flags(metadata: &Metadata, package: &Package, target: &Target) -> Result<Vec<String>> {
304 : // smoelius: The following comments are old and retained for posterity. The linking flags are
305 : // now initialized using a `OnceCell`, which makes the mutex unnecessary.
306 : // smoelius: Force rebuilding of the example by removing it. This is kind of messy. The
307 : // example is a shared resource that may be needed by multiple tests. For now, I lock a mutex
308 : // while the example is removed and put back.
309 : // smoelius: Should we use a temporary target directory here?
310 0 : let output = {
311 0 : remove_example(metadata, package, target)?;
312 :
313 : // smoelius: Because of lazy initialization, `cargo build` is run only once. Seeing
314 : // "Building example `target`" for one example but not for others is confusing. So instead
315 : // say "Building `package` examples".
316 0 : dylint_internal::cargo::build(&format!("`{}` examples", package.name))
317 0 : .build()
318 0 : .envs([(env::CARGO_TERM_COLOR, "never")])
319 0 : .args([
320 0 : "--manifest-path",
321 0 : package.manifest_path.as_ref(),
322 0 : "--example",
323 0 : &target.name,
324 0 : "--verbose",
325 0 : ])
326 0 : .logged_output(true)?
327 : };
328 :
329 0 : let matches = output
330 0 : .stderr
331 0 : .lines()
332 0 : .map(|line| {
333 0 : let line =
334 0 : line.with_context(|| format!("Could not read from `{}`", package.manifest_path))?;
335 0 : Ok((*RE).captures(&line).and_then(|captures| {
336 0 : let args = captures[1]
337 0 : .split(' ')
338 0 : .map(ToOwned::to_owned)
339 0 : .collect::<Vec<_>>();
340 0 : if args.first().map_or(false, is_rustc)
341 0 : && args
342 0 : .as_slice()
343 0 : .windows(2)
344 0 : .any(|window| window == ["--crate-name", &snake_case(&target.name)])
345 : {
346 0 : Some(args)
347 : } else {
348 0 : None
349 : }
350 0 : }))
351 0 : })
352 0 : .collect::<Result<Vec<Option<Vec<_>>>>>()?;
353 :
354 0 : let mut matches = matches.into_iter().flatten().collect::<Vec<Vec<_>>>();
355 0 : ensure!(
356 0 : matches.len() <= 1,
357 0 : "Found multiple `rustc` invocations for `{}`",
358 : target.name
359 : );
360 0 : matches
361 0 : .pop()
362 0 : .ok_or_else(|| anyhow!("Found no `rustc` invocations for `{}`", target.name))
363 0 : }
364 :
365 0 : fn remove_example(metadata: &Metadata, _package: &Package, target: &Target) -> Result<()> {
366 0 : let examples = metadata.target_directory.join("debug/examples");
367 0 : for entry in
368 0 : read_dir(&examples).with_context(|| format!("`read_dir` failed for `{examples}`"))?
369 : {
370 0 : let entry = entry.with_context(|| format!("`read_dir` failed for `{examples}`"))?;
371 0 : let path = entry.path();
372 :
373 0 : if let Some(file_name) = path.file_name() {
374 0 : let s = file_name.to_string_lossy();
375 0 : let target_name = snake_case(&target.name);
376 0 : if s == target_name.clone() + consts::EXE_SUFFIX
377 0 : || s.starts_with(&(target_name.clone() + "-"))
378 : {
379 0 : remove_file(&path).with_context(|| {
380 0 : format!("`remove_file` failed for `{}`", path.to_string_lossy())
381 0 : })?;
382 0 : }
383 0 : }
384 : }
385 :
386 0 : Ok(())
387 0 : }
388 :
389 0 : fn next<I, T>(flag: &str, iter: &mut I) -> Result<T>
390 0 : where
391 0 : I: Iterator<Item = T>,
392 0 : {
393 0 : iter.next()
394 0 : .ok_or_else(|| anyhow!("Missing argument for `{}`", flag))
395 0 : }
396 :
397 0 : fn copy_with_extension<P: AsRef<Path>, Q: AsRef<Path>>(
398 0 : from: P,
399 0 : to: Q,
400 0 : extension: &str,
401 0 : ) -> Result<u64> {
402 0 : let from = from.as_ref().with_extension(extension);
403 0 : let to = to.as_ref().with_extension(extension);
404 0 : copy(from, to).map_err(Into::into)
405 0 : }
406 :
407 : static MUTEX: Mutex<()> = Mutex::new(());
408 :
409 0 : fn run_tests(driver: &Path, src_base: &Path, config: &ui::Config) {
410 0 : let _lock = MUTEX.lock().unwrap();
411 0 :
412 0 : // smoelius: There doesn't seem to be a way to set environment variables using `compiletest`'s
413 0 : // [`Config`](https://docs.rs/compiletest_rs/0.7.1/compiletest_rs/common/struct.Config.html)
414 0 : // struct. For comparison, where Clippy uses `compiletest`, it sets environment variables
415 0 : // directly (see: https://github.com/rust-lang/rust-clippy/blob/master/tests/compile-test.rs).
416 0 : //
417 0 : // Of course, even if `compiletest` had such support, it would need to be incorporated into
418 0 : // `dylint_testing`.
419 0 :
420 0 : let _var = config
421 0 : .dylint_toml
422 0 : .as_ref()
423 0 : .map(|value| VarGuard::set(env::DYLINT_TOML, value));
424 :
425 0 : let config = compiletest::Config {
426 0 : mode: compiletest::common::Mode::Ui,
427 0 : rustc_path: driver.to_path_buf(),
428 0 : src_base: src_base.to_path_buf(),
429 0 : target_rustcflags: Some(
430 0 : config.rustc_flags.clone().join(" ")
431 0 : + " --emit=metadata"
432 0 : + if cfg!(feature = "deny_warnings") {
433 0 : " -Dwarnings"
434 : } else {
435 0 : ""
436 : }
437 0 : + " -Zui-testing",
438 0 : ),
439 0 : ..compiletest::Config::default()
440 0 : };
441 0 :
442 0 : compiletest::run_tests(&config);
443 0 : }
444 :
445 : // smoelius: `VarGuard` was copied from:
446 : // https://github.com/rust-lang/rust-clippy/blob/9cc8da222b3893bc13bc13c8827e93f8ea246854/tests/compile-test.rs
447 :
448 : /// Restores an env var on drop
449 : #[must_use]
450 : struct VarGuard {
451 : key: &'static str,
452 : value: Option<OsString>,
453 : }
454 :
455 : impl VarGuard {
456 0 : fn set(key: &'static str, val: impl AsRef<OsStr>) -> Self {
457 0 : let value = var_os(key);
458 0 : set_var(key, val);
459 0 : Self { key, value }
460 0 : }
461 : }
462 :
463 : impl Drop for VarGuard {
464 0 : fn drop(&mut self) {
465 0 : match self.value.as_deref() {
466 0 : None => remove_var(self.key),
467 0 : Some(value) => set_var(self.key, value),
468 : }
469 0 : }
470 : }
471 :
472 0 : fn snake_case(name: &str) -> String {
473 0 : name.replace('-', "_")
474 0 : }
|
