Tweak docker, make things easier for new contributors

README.md

@@ -48,23 +48,30 @@ of its output is into the `./work` directory, where it maintains its
 own rustup installation, crate mirrors, etc.
 
 ```
-cd docker && docker build . && cd ..    # This takes a while
-cargo run -- prepare-local
-cargo run -- define-ex stable beta
+cargo run -- prepare-local --docker-env mini
+cargo run -- define-ex --crate-select=demo stable beta
 cargo run -- prepare-ex
 cargo run -- run
 cargo run -- gen-report work/ex/default/
 ```
 
 This will output a report to `./work/ex/default/index.html`.
 
-Here's what each of these steps does:
+Delete things with
+```
+cargo run -- delete-all-target-dirs
+cargo run -- delete-all-results
+cargo run -- delete-ex
+```
+Each command except `prepare-local` optionally takes an `--ex` argument
+to identify the experiment being referred to. If not supplied, this
+defaults to `default`. Here's what each of the steps does:
 
 * `prepare-local` - sets up the stable toolchain for internal use,
   builds the docker container, builds lists of crates. This needs to
   be rerun periodically, but not between every experiment.
 
-* `define-ex` - defines a new experiment, by default named 'default',
+* `define-ex` - defines a new experiment
   performing a build-test experiment on the 'demo' set of crates.
 
 * `prepare-ex` - fetches repos from github and captures their commit
@@ -77,6 +84,9 @@ Here's what each of these steps does:
 * `gen-report` - summarize the experiment results to
   work/ex/default/index.html
 
+* `delete-all-target-dirs`/`delete-all-results`/`delete-ex` - clean up
+  everything relating to this experiment
+
 ### Custom toolchains
 
 Toolchains for rust PRs that have been built by by asking bors to try a PR can
@@ -195,7 +205,7 @@ the sheet that does not have a status of 'Complete' or 'Failed'.
        and `BETA_DATE` is the date from
        `curl -sSL static.rust-lang.org/dist/channel-rust-beta.toml | grep '^date ='` (it is *not*
        necessarily the same date as retrieved in the `BETA_VERSION` command).
-   - Run `cargo run --release -- define-ex --ex EX_NAME EX_START EX_END --crate-select=full`.
+   - Run `cargo run --release -- define-ex --crate-select=full --ex EX_NAME EX_START EX_END`.
      This will complete in a few seconds.
    - Run `cargo run --release -- prepare-ex --ex EX_NAME`.
    - Change status to 'Preparing'.

docker/Dockerfile → docker/Dockerfile.full

@@ -1,18 +1,17 @@
 FROM ubuntu:16.04
 
 ENV LAST_UPDATE=2017-05-25
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update
 
 # Tools
-RUN apt-get install -y gcc
-RUN apt-get install -y pkg-config
-RUN apt-get install -y cmake
+RUN apt-get install -y gcc pkg-config cmake
 
 # Native deps
 RUN apt-get install -y libssl-dev
 # Hopefully this pulls in lots of stuff
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --install-suggests libgtk-3-dev
+RUN apt-get install -y --install-suggests libgtk-3-dev
 
 WORKDIR /source
 
@@ -22,11 +21,9 @@ WORKDIR /source
 # re https://github.com/docker/docker/issues/7198#issuecomment-158566258
 RUN adduser --no-create-home --disabled-login --gecos "" crater --uid 1000
 
-# The run.sh script configures the user id, controlled by -e USER_ID, and then
+# Configure the user id, controlled by -e USER_ID, and then
 # runs some command, controlled by -e CMD
-COPY run.sh /run.sh
-CMD ["/run.sh"]
+CMD ["bash", "-c", "usermod -u \"$USER_ID\" crater && exec su crater -c \"/run2.sh $CMD\""]
 
-# run.sh script runs run2.sh under su, which sets up the environment and runs
-# $CMD
+# sets up the environment for $CMD
 COPY run2.sh /run2.sh

docker/Dockerfile.mini

@@ -0,0 +1,27 @@
+FROM ubuntu:16.04
+
+ENV LAST_UPDATE=2017-05-25
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+
+# Tools
+RUN apt-get install -y gcc pkg-config cmake
+
+# Native deps
+RUN apt-get install -y libssl-dev
+
+WORKDIR /source
+
+# Create a crater user that run.sh will map to the value of the host user via
+# the USER_ID environment variable, to make the files the container writes not
+# be owned by root, but by the running host user
+# re https://github.com/docker/docker/issues/7198#issuecomment-158566258
+RUN adduser --no-create-home --disabled-login --gecos "" crater --uid 1000
+
+# Configure the user id, controlled by -e USER_ID, and then
+# runs some command, controlled by -e CMD
+CMD ["bash", "-c", "usermod -u \"$USER_ID\" crater && exec su crater -c \"/run2.sh $CMD\""]
+
+# sets up the environment for $CMD
+COPY run2.sh /run2.sh

src/cli.rs

@@ -34,14 +34,17 @@ use std::str::FromStr;
 #[derive(Debug, Clone)]
 pub struct Ex(String);
 
+#[derive(Debug, Clone)]
+pub struct DockerEnv(String);
+
 #[derive(Debug, Clone)]
 pub struct Dest(PathBuf);
 
 pub trait Cmd {
     fn run(&self) -> Result<()>;
 }
 
-struct PrepareLocal;
+struct PrepareLocal(DockerEnv);
 struct DefineEx(Ex, Toolchain, Toolchain, ExMode, ExCrateSelect);
 #[derive(StructOpt)]
 #[structopt(name = "prepare-ex", about = "prepare shared and local data for experiment")]
@@ -91,9 +94,10 @@ struct Serve;
 // Local prep
 impl Cmd for PrepareLocal {
     fn run(&self) -> Result<()> {
+        let docker_env = &(self.0).0;
         let stable_tc = Toolchain::Dist("stable".into());
         stable_tc.prepare()?;
-        docker::build_container()?;
+        docker::build_container(docker_env)?;
         lists::create_all_lists(false)
     }
 }
@@ -278,7 +282,7 @@ pub mod conv {
             cmd(
                 "prepare-local",
                 "acquire toolchains, build containers, build crate lists",
-            ),
+            ).arg(opt("docker-env", "full")),
             // List creation
             cmd("create-lists", "create all the lists of crates"),
             // Master experiment prep
@@ -351,9 +355,13 @@ pub mod conv {
                 .parse::<ExCrateSelect>()
         }
 
+        fn docker_env(m: &ArgMatches) -> DockerEnv {
+            DockerEnv(m.value_of("docker-env").expect("").to_string())
+        }
+
         Ok(match m.subcommand() {
             // Local prep
-            ("prepare-local", _) => Box::new(PrepareLocal),
+            ("prepare-local", Some(m)) => Box::new(PrepareLocal(docker_env(m))),
             ("create-lists", _) => Box::new(CreateLists),
 
             // Master experiment prep

src/docker.rs

@@ -10,8 +10,13 @@ static IMAGE_NAME: &'static str = "crater";
 /// Builds the docker container image, 'crater', what will be used
 /// to isolate builds from each other. This expects the Dockerfile
 /// to exist in the `docker` directory, at runtime.
-pub fn build_container() -> Result<()> {
-    run::run("docker", &["build", "-t", IMAGE_NAME, "docker"], &[])
+pub fn build_container(docker_env: &str) -> Result<()> {
+    let dockerfile = format!("docker/Dockerfile.{}", docker_env);
+    run::run(
+        "docker",
+        &["build", "-f", &dockerfile, "-t", IMAGE_NAME, "docker"],
+        &[],
+    )
 }
 
 #[derive(Copy, Clone)]

src/report/mod.rs

@@ -87,8 +87,9 @@ fn write_logs<W: ReportWriter>(ex: &ex::Experiment, dest: &W) -> Result<()> {
     let db = FileDB::for_experiment(ex);
     let crates = ex.crates()?;
     let num_crates = crates.len();
+    let progress_every = (num_crates / PROGRESS_FRACTION) + 1;
     for (i, krate) in crates.into_iter().enumerate() {
-        if i % (num_crates / PROGRESS_FRACTION) == 0 {
+        if i % progress_every == 0 {
             info!("wrote logs for {}/{} crates", i, num_crates)
         }
         for tc in &ex.toolchains {