From 6ff021cb7f26e49d8895a149a5e10ef2312133f9 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 7 Mar 2021 11:49:28 +0100 Subject: [PATCH 1/4] add simple test for building docs for a crate --- src/docbuilder/rustwide_builder.rs | 16 ++++++++++++++++ src/test/mod.rs | 16 +++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 1f2402d13..a2bbf0735 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -753,3 +753,19 @@ pub(crate) struct BuildResult { pub(crate) docsrs_version: String, pub(crate) successful: bool, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test::wrapper; + + #[test] + fn test_build_crate() { + wrapper(|env| { + let mut builder = RustwideBuilder::init(env).unwrap(); + builder + .build_package("log", "0.4.14", PackageKind::CratesIo) + .map(|_| ()) + }) + } +} diff --git a/src/test/mod.rs b/src/test/mod.rs index 5e839ecc4..6bc3a6e82 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -13,6 +13,7 @@ use reqwest::{ blocking::{Client, RequestBuilder}, Method, }; +use std::fs; use std::{panic, sync::Arc}; pub(crate) fn wrapper(f: impl FnOnce(&TestEnvironment) -> Result<(), Error>) { @@ -103,10 +104,12 @@ pub(crate) struct TestEnvironment { } pub(crate) fn init_logger() { - // If this fails it's probably already initialized - let _ = env_logger::from_env(env_logger::Env::default().filter("DOCSRS_LOG")) - .is_test(true) - .try_init(); + // initializing rustwide logging also sets the global logger + rustwide::logging::init_with( + env_logger::from_env(env_logger::Env::default().filter("DOCSRS_LOG")) + .is_test(true) + .build(), + ); } impl TestEnvironment { @@ -137,8 +140,11 @@ impl TestEnvironment { fn base_config(&self) -> Config { let mut config = Config::from_env().expect("failed to get base config"); + // create index directory + fs::create_dir_all(config.registry_index_path.clone()).unwrap(); + // Use less connections for each test compared to production. - config.max_pool_size = 2; + config.max_pool_size = 4; config.min_pool_idle = 0; // Use the database for storage, as it's faster than S3. From de27a1bcbd5b80f04213005124b0c5d27a3700b5 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 7 Mar 2021 17:58:43 +0100 Subject: [PATCH 2/4] extend build-test with assertions --- .github/workflows/ci.yml | 7 ++- src/docbuilder/rustwide_builder.rs | 85 ++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8eb2be917..f1a7330b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,9 +60,14 @@ jobs: - name: Build run: cargo build --workspace --locked - - name: Test + - name: fast tests run: cargo test --workspace --locked + - name: slow tests + env: + DOCSRS_INCLUDE_DEFAULT_TARGETS: true + run: cargo test --workspace --locked -- --ignored + - name: Clean up the database run: docker-compose down --volumes diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index a2bbf0735..4a5d9ecfb 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -757,15 +757,94 @@ pub(crate) struct BuildResult { #[cfg(test)] mod tests { use super::*; - use crate::test::wrapper; + use crate::test::{assert_redirect, assert_success, wrapper}; #[test] + #[ignore] fn test_build_crate() { wrapper(|env| { + let crate_ = "log"; + let version = "0.4.14"; + let default_target = "x86_64-unknown-linux-gnu"; + + assert_eq!(env.config().include_default_targets, true); + let mut builder = RustwideBuilder::init(env).unwrap(); builder - .build_package("log", "0.4.14", PackageKind::CratesIo) - .map(|_| ()) + .build_package(crate_, version, PackageKind::CratesIo) + .map(|_| ())?; + + // check release record in the db (default and other targets) + let mut conn = env.db().conn(); + let rows = conn + .query( + "SELECT + r.rustdoc_status, + r.default_target, + r.doc_targets + FROM + crates as c + INNER JOIN releases AS r ON c.id = r.crate_id + WHERE + c.name = $1 AND + r.version = $2", + &[&crate_, &version], + ) + .unwrap(); + let row = rows.get(0).unwrap(); + + assert_eq!(row.get::<_, bool>("rustdoc_status"), true); + assert_eq!(row.get::<_, String>("default_target"), default_target); + + let mut targets: Vec = row + .get::<_, Value>("doc_targets") + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap().to_owned()) + .collect(); + targets.sort(); + assert_eq!( + targets, + vec![ + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + ] + ); + + let storage = env.storage(); + let web = env.frontend(); + + let base = format!("rustdoc/{}/{}", crate_, version); + + // default target was built and is accessible + assert!(storage.exists(&format!("{}/{}/index.html", base, crate_))?); + assert_success(&format!("/{0}/{1}/{0}", crate_, version), web)?; + + // other targets too + for target in DEFAULT_TARGETS { + let target_docs_present = + storage.exists(&format!("{}/{}/{}/index.html", base, target, crate_))?; + + let target_url = format!("/{0}/{1}/{2}/{0}/index.html", crate_, version, target); + + if target == &default_target { + assert!(!target_docs_present); + assert_redirect( + &target_url, + &format!("/{0}/{1}/{0}/index.html", crate_, version), + web, + )?; + } else { + assert!(target_docs_present); + assert_success(&target_url, web)?; + } + } + + Ok(()) }) } } From 1b5c3374fbbcc4ea1aeaa4d8ff5291ff93995b25 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Mon, 8 Mar 2021 16:07:23 +0100 Subject: [PATCH 3/4] build-test: use local small build-env, dummy crate for test --- .github/workflows/ci.yml | 5 +++++ dockerfiles/Dockerfile-small-build-env | 6 ++++++ src/docbuilder/rustwide_builder.rs | 18 +++++++++++------- 3 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 dockerfiles/Dockerfile-small-build-env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1a7330b3..45b4d4ed8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,9 +63,14 @@ jobs: - name: fast tests run: cargo test --workspace --locked + - name: create small build-environment + run: | + docker build -t buildenv - < dockerfiles/Dockerfile-small-build-env + - name: slow tests env: DOCSRS_INCLUDE_DEFAULT_TARGETS: true + DOCS_RS_LOCAL_DOCKER_IMAGE: buildenv run: cargo test --workspace --locked -- --ignored - name: Clean up the database diff --git a/dockerfiles/Dockerfile-small-build-env b/dockerfiles/Dockerfile-small-build-env new file mode 100644 index 000000000..118b375fc --- /dev/null +++ b/dockerfiles/Dockerfile-small-build-env @@ -0,0 +1,6 @@ +FROM ubuntu:focal + +RUN apt-get update +RUN apt-get install -y --no-install-recommends apt-utils build-essential sudo git + +CMD ["bash"] diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 4a5d9ecfb..1e617d5b3 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -763,8 +763,9 @@ mod tests { #[ignore] fn test_build_crate() { wrapper(|env| { - let crate_ = "log"; - let version = "0.4.14"; + let crate_ = DUMMY_CRATE_NAME; + let crate_path = crate_.replace("-", "_"); + let version = DUMMY_CRATE_VERSION; let default_target = "x86_64-unknown-linux-gnu"; assert_eq!(env.config().include_default_targets, true); @@ -821,21 +822,24 @@ mod tests { let base = format!("rustdoc/{}/{}", crate_, version); // default target was built and is accessible - assert!(storage.exists(&format!("{}/{}/index.html", base, crate_))?); - assert_success(&format!("/{0}/{1}/{0}", crate_, version), web)?; + assert!(storage.exists(&format!("{}/{}/index.html", base, crate_path))?); + assert_success(&format!("/{}/{}/{}", crate_, version, crate_path), web)?; // other targets too for target in DEFAULT_TARGETS { let target_docs_present = - storage.exists(&format!("{}/{}/{}/index.html", base, target, crate_))?; + storage.exists(&format!("{}/{}/{}/index.html", base, target, crate_path))?; - let target_url = format!("/{0}/{1}/{2}/{0}/index.html", crate_, version, target); + let target_url = format!( + "/{}/{}/{}/{}/index.html", + crate_, version, target, crate_path + ); if target == &default_target { assert!(!target_docs_present); assert_redirect( &target_url, - &format!("/{0}/{1}/{0}/index.html", crate_, version), + &format!("/{}/{}/{}/index.html", crate_, version, crate_path), web, )?; } else { From 050b8a3d280095b36118849959bc3255daa50962 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 9 Mar 2021 06:12:40 +0100 Subject: [PATCH 4/4] add fast_init=true to rustwide workspace builder --- src/docbuilder/rustwide_builder.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 1e617d5b3..b62e75536 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -87,6 +87,9 @@ impl RustwideBuilder { if let Some(custom_image) = &config.local_docker_image { builder = builder.sandbox_image(SandboxImage::local(&custom_image)?); } + if cfg!(test) { + builder = builder.fast_init(true); + } let workspace = builder.init()?; workspace.purge_all_build_dirs()?;