Auto merge of #30629 - brson:emscripten-upstream, r=alexcrichton

Here's another go at adding emscripten support. This needs to wait again on new [libc definitions](rust-lang/libc#122) landing. To get the libc definitions right I had to add support for i686-unknown-linux-musl, which are very similar to emscripten's, which are derived from arm/musl.

This branch additionally removes the makefile dependency on the `EMSCRIPTEN` environment variable by not building the unused compiler-rt.

Again, this is not sufficient for actually compiling to asmjs since it needs additional LLVM patches.

r? @alexcrichton

configure

@@ -1305,6 +1305,12 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
             putvar CFG_DISABLE_JEMALLOC
             ;;
 
+        *-emscripten)
+            step_msg "targeting emscripten, disabling jemalloc"
+            CFG_DISABLE_JEMALLOC=1
+            putvar CFG_DISABLE_JEMALLOC
+            ;;
+
         *)
             ;;
     esac

mk/cfg/asmjs-unknown-emscripten.mk

@@ -0,0 +1,23 @@
+# asmjs-unknown-emscripten configuration
+CC_asmjs-unknown-emscripten=emcc
+CXX_asmjs-unknown-emscripten=em++
+CPP_asmjs-unknown-emscripten=$(CPP)
+AR_asmjs-unknown-emscripten=emar
+CFG_LIB_NAME_asmjs-unknown-emscripten=lib$(1).so
+CFG_STATIC_LIB_NAME_asmjs-unknown-emscripten=lib$(1).a
+CFG_LIB_GLOB_asmjs-unknown-emscripten=lib$(1)-*.so
+CFG_LIB_DSYM_GLOB_asmjs-unknown-emscripten=lib$(1)-*.dylib.dSYM
+CFG_JEMALLOC_CFLAGS_asmjs-unknown-emscripten := -m32 $(CFLAGS)
+CFG_GCCISH_CFLAGS_asmjs-unknown-emscripten := -Wall -Werror -g -fPIC -m32 $(CFLAGS)
+CFG_GCCISH_CXXFLAGS_asmjs-unknown-emscripten := -fno-rtti $(CXXFLAGS)
+CFG_GCCISH_LINK_FLAGS_asmjs-unknown-emscripten := -shared -fPIC -ldl -pthread  -lrt -g -m32
+CFG_GCCISH_DEF_FLAG_asmjs-unknown-emscripten := -Wl,--export-dynamic,--dynamic-list=
+CFG_LLC_FLAGS_asmjs-unknown-emscripten :=
+CFG_INSTALL_NAME_asmjs-unknown-emscripten =
+CFG_EXE_SUFFIX_asmjs-unknown-emscripten =
+CFG_WINDOWSY_asmjs-unknown-emscripten :=
+CFG_UNIXY_asmjs-unknown-emscripten := 1
+CFG_LDPATH_asmjs-unknown-emscripten :=
+CFG_RUN_asmjs-unknown-emscripten=$(2)
+CFG_RUN_TARG_asmjs-unknown-emscripten=$(call CFG_RUN_asmjs-unknown-emscripten,,$(2))
+CFG_GNU_TRIPLE_asmjs-unknown-emscripten := asmjs-unknown-emscripten

mk/cfg/i686-unknown-linux-musl.mk

@@ -0,0 +1,28 @@
+# i686-unknown-linux-musl configuration
+CC_i686-unknown-linux-musl=$(CFG_MUSL_ROOT)/bin/musl-gcc
+CXX_i686-unknown-linux-musl=notaprogram
+CPP_i686-unknown-linux-musl=$(CFG_MUSL_ROOT)/bin/musl-gcc -E
+AR_i686-unknown-linux-musl=$(AR)
+CFG_INSTALL_ONLY_RLIB_i686-unknown-linux-musl = 1
+CFG_LIB_NAME_i686-unknown-linux-musl=lib$(1).so
+CFG_STATIC_LIB_NAME_i686-unknown-linux-musl=lib$(1).a
+CFG_LIB_GLOB_i686-unknown-linux-musl=lib$(1)-*.so
+CFG_JEMALLOC_CFLAGS_i686-unknown-linux-musl := -m32 -Wl,-melf_i386
+CFG_GCCISH_CFLAGS_i686-unknown-linux-musl := -Wall -Werror -g -fPIC -m32 -Wl,-melf_i386
+CFG_GCCISH_CXXFLAGS_i686-unknown-linux-musl :=
+CFG_GCCISH_LINK_FLAGS_i686-unknown-linux-musl :=
+CFG_GCCISH_DEF_FLAG_i686-unknown-linux-musl :=
+CFG_LLC_FLAGS_i686-unknown-linux-musl :=
+CFG_INSTALL_NAME_i686-unknown-linux-musl =
+CFG_EXE_SUFFIX_i686-unknown-linux-musl =
+CFG_WINDOWSY_i686-unknown-linux-musl :=
+CFG_UNIXY_i686-unknown-linux-musl := 1
+CFG_LDPATH_i686-unknown-linux-musl :=
+CFG_RUN_i686-unknown-linux-musl=$(2)
+CFG_RUN_TARG_i686-unknown-linux-musl=$(call CFG_RUN_i686-unknown-linux-musl,,$(2))
+CFG_GNU_TRIPLE_i686-unknown-linux-musl := i686-unknown-linux-musl
+CFG_THIRD_PARTY_OBJECTS_i686-unknown-linux-musl := crt1.o crti.o crtn.o
+CFG_INSTALLED_OBJECTS_i686-unknown-linux-musl := crt1.o crti.o crtn.o
+
+NATIVE_DEPS_libc_T_i686-unknown-linux-musl += libc.a
+NATIVE_DEPS_std_T_i686-unknown-linux-musl += libunwind.a crt1.o crti.o crtn.o

mk/main.mk

@@ -361,6 +361,9 @@ export CFG_DISABLE_UNSTABLE_FEATURES
 export RUSTC_BOOTSTRAP_KEY:=$(CFG_BOOTSTRAP_KEY)
 endif
 export CFG_BOOTSTRAP_KEY
+ifdef CFG_MUSL_ROOT
+export CFG_MUSL_ROOT
+endif
 
 ######################################################################
 # Per-stage targets and runner

mk/rt.mk

@@ -254,6 +254,15 @@ ifeq ($$(findstring freebsd,$(1)),freebsd)
 	COMPRT_CFLAGS_$(1) += -I/usr/include/c++/v1
 endif
 
+ifeq ($$(findstring emscripten,$(1)),emscripten)
+
+# FIXME: emscripten doesn't use compiler-rt and can't build it without
+# further hacks
+$$(COMPRT_LIB_$(1)):
+	touch $$@
+
+else
+
 $$(COMPRT_LIB_$(1)): $$(COMPRT_DEPS) $$(MKFILE_DEPS)
 	@$$(call E, make: compiler-rt)
 	$$(Q)$$(MAKE) -C "$(S)src/compiler-rt" \
@@ -266,7 +275,10 @@ $$(COMPRT_LIB_$(1)): $$(COMPRT_DEPS) $$(MKFILE_DEPS)
 		TargetTriple=$(1) \
 		triple-builtins
 	$$(Q)cp $$(COMPRT_BUILD_DIR_$(1))/triple/builtins/libcompiler_rt.a $$@
+
+endif # if emscripten
 endif
+
 ################################################################################
 # libbacktrace
 #
@@ -301,6 +313,12 @@ $$(BACKTRACE_LIB_$(1)):
 	touch $$@
 else
 
+ifeq ($$(findstring emscripten,$(1)),emscripten)
+# FIXME: libbacktrace doesn't understand the emscripten triple
+$$(BACKTRACE_LIB_$(1)):
+	touch $$@
+else
+
 ifdef CFG_ENABLE_FAST_MAKE
 BACKTRACE_DEPS := $(S)/.gitmodules
 else
@@ -348,6 +366,7 @@ $$(BACKTRACE_LIB_$(1)): $$(BACKTRACE_BUILD_DIR_$(1))/Makefile $$(MKFILE_DEPS)
 		INCDIR=$(S)src/libbacktrace
 	$$(Q)cp $$(BACKTRACE_BUILD_DIR_$(1))/.libs/libbacktrace.a $$@
 
+endif # endif for emscripten
 endif # endif for msvc
 endif # endif for ios
 endif # endif for darwin

src/compiletest/runtest.rs

@@ -1357,7 +1357,12 @@ fn make_lib_name(config: &Config, auxfile: &Path, testfile: &Path) -> PathBuf {
 
 fn make_exe_name(config: &Config, testfile: &Path) -> PathBuf {
     let mut f = output_base_name(config, testfile);
-    if !env::consts::EXE_SUFFIX.is_empty() {
+    // FIXME: This is using the host architecture exe suffix, not target!
+    if config.target == "asmjs-unknown-emscripten" {
+        let mut fname = f.file_name().unwrap().to_os_string();
+        fname.push(".js");
+        f.set_file_name(&fname);
+    } else if !env::consts::EXE_SUFFIX.is_empty() {
         let mut fname = f.file_name().unwrap().to_os_string();
         fname.push(env::consts::EXE_SUFFIX);
         f.set_file_name(&fname);
@@ -1370,6 +1375,12 @@ fn make_run_args(config: &Config, props: &TestProps, testfile: &Path)
     // If we've got another tool to run under (valgrind),
     // then split apart its command
     let mut args = split_maybe_args(&config.runtool);
+
+    // If this is emscripten, then run tests under nodejs
+    if config.target == "asmjs-unknown-emscripten" {
+        args.push("nodejs".to_owned());
+    }
+
     let exe_file = make_exe_name(config, testfile);
 
     // FIXME (#9639): This needs to handle non-utf8 paths

src/compiletest/util.rs

@@ -26,6 +26,7 @@ const OS_TABLE: &'static [(&'static str, &'static str)] = &[
     ("win32", "windows"),
     ("windows", "windows"),
     ("solaris", "solaris"),
+    ("emscripten", "emscripten"),
 ];
 
 const ARCH_TABLE: &'static [(&'static str, &'static str)] = &[
@@ -44,6 +45,7 @@ const ARCH_TABLE: &'static [(&'static str, &'static str)] = &[
     ("sparc", "sparc"),
     ("x86_64", "x86_64"),
     ("xcore", "xcore"),
+    ("asmjs", "asmjs"),
 ];
 
 pub fn get_os(triple: &str) -> &'static str {

src/liballoc_system/lib.rs

@@ -30,7 +30,8 @@ extern crate libc;
               target_arch = "arm",
               target_arch = "mips",
               target_arch = "powerpc",
-              target_arch = "powerpc64")))]
+              target_arch = "powerpc64",
+              target_arch = "asmjs")))]
 const MIN_ALIGN: usize = 8;
 #[cfg(all(any(target_arch = "x86_64",
               target_arch = "aarch64")))]

src/librustc_back/target/asmjs_unknown_emscripten.rs

@@ -0,0 +1,38 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use super::{Target, TargetOptions};
+
+pub fn target() -> Target {
+    let opts = TargetOptions {
+        linker: "emcc".to_string(),
+        ar: "emar".to_string(),
+
+        dynamic_linking: false,
+        executables: true,
+        exe_suffix: ".js".to_string(),
+        no_compiler_rt: true,
+        linker_is_gnu: true,
+        allow_asm: false,
+        archive_format: "gnu".to_string(),
+        obj_is_bitcode: true,
+        .. Default::default()
+    };
+    Target {
+        llvm_target: "asmjs-unknown-emscripten".to_string(),
+        target_endian: "little".to_string(),
+        target_pointer_width: "32".to_string(),
+        target_os: "emscripten".to_string(),
+        target_env: "".to_string(),
+        target_vendor: "unknown".to_string(),
+        arch: "asmjs".to_string(),
+        options: opts,
+    }
+}

src/librustc_back/target/i686_unknown_linux_musl.rs

@@ -0,0 +1,46 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// See x86_64_unknown_linux_musl for explanation of arguments
+
+use target::Target;
+
+pub fn target() -> Target {
+    let mut base = super::linux_base::opts();
+    base.cpu = "pentium4".to_string();
+    base.pre_link_args.push("-m32".to_string());
+    base.pre_link_args.push("-Wl,-melf_i386".to_string());
+
+    base.pre_link_args.push("-nostdlib".to_string());
+    base.pre_link_args.push("-static".to_string());
+    base.pre_link_args.push("-Wl,--eh-frame-hdr".to_string());
+
+    base.pre_link_args.push("-Wl,-(".to_string());
+    base.post_link_args.push("-Wl,-)".to_string());
+
+    base.pre_link_objects_exe.push("crt1.o".to_string());
+    base.pre_link_objects_exe.push("crti.o".to_string());
+    base.post_link_objects.push("crtn.o".to_string());
+
+    base.dynamic_linking = false;
+    base.has_rpath = false;
+    base.position_independent_executables = false;
+
+    Target {
+        llvm_target: "i686-unknown-linux-musl".to_string(),
+        target_endian: "little".to_string(),
+        target_pointer_width: "32".to_string(),
+        arch: "x86".to_string(),
+        target_os: "linux".to_string(),
+        target_env: "musl".to_string(),
+        target_vendor: "unknown".to_string(),
+        options: base,
+    }
+}

src/librustc_back/target/mod.rs

@@ -204,6 +204,10 @@ pub struct TargetOptions {
     /// Flag indicating whether ELF TLS (e.g. #[thread_local]) is available for
     /// this target.
     pub has_elf_tls: bool,
+    // This is mainly for easy compatibility with emscripten.
+    // If we give emcc .o files that are actually .bc files it
+    // will 'just work'.
+    pub obj_is_bitcode: bool,
 }
 
 impl Default for TargetOptions {
@@ -251,6 +255,7 @@ impl Default for TargetOptions {
             exe_allocation_crate: "alloc_system".to_string(),
             allow_asm: true,
             has_elf_tls: false,
+            obj_is_bitcode: false,
         }
     }
 }
@@ -426,6 +431,7 @@ impl Target {
             armv7_unknown_linux_gnueabihf,
             aarch64_unknown_linux_gnu,
             x86_64_unknown_linux_musl,
+            i686_unknown_linux_musl,
             mips_unknown_linux_musl,
             mipsel_unknown_linux_musl,
 
@@ -461,7 +467,8 @@ impl Target {
             x86_64_pc_windows_msvc,
             i686_pc_windows_msvc,
 
-            le32_unknown_nacl
+            le32_unknown_nacl,
+            asmjs_unknown_emscripten
         );
 
 

src/librustc_trans/back/write.rs

@@ -244,7 +244,6 @@ pub struct ModuleConfig {
     emit_ir: bool,
     emit_asm: bool,
     emit_obj: bool,
-
     // Miscellaneous flags.  These are mostly copied from command-line
     // options.
     no_verify: bool,
@@ -254,7 +253,11 @@ pub struct ModuleConfig {
     vectorize_loop: bool,
     vectorize_slp: bool,
     merge_functions: bool,
-    inline_threshold: Option<usize>
+    inline_threshold: Option<usize>,
+    // Instead of creating an object file by doing LLVM codegen, just
+    // make the object file bitcode. Provides easy compatibility with
+    // emscripten's ecc compiler, when used as the linker.
+    obj_is_bitcode: bool,
 }
 
 unsafe impl Send for ModuleConfig { }
@@ -272,6 +275,7 @@ impl ModuleConfig {
             emit_ir: false,
             emit_asm: false,
             emit_obj: false,
+            obj_is_bitcode: false,
 
             no_verify: false,
             no_prepopulate_passes: false,
@@ -290,6 +294,7 @@ impl ModuleConfig {
         self.no_builtins = trans.no_builtins;
         self.time_passes = sess.time_passes();
         self.inline_threshold = sess.opts.cg.inline_threshold;
+        self.obj_is_bitcode = sess.target.target.options.obj_is_bitcode;
 
         // Copy what clang does by turning on loop vectorization at O2 and
         // slp vectorization at O3. Otherwise configure other optimization aspects
@@ -530,11 +535,21 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext,
         f(cpm);
     }
 
-    if config.emit_bc {
-        let ext = format!("{}.bc", name_extra);
-        let out = output_names.with_extension(&ext);
-        let out = path2cstr(&out);
-        llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr());
+    // Change what we write and cleanup based on whether obj files are
+    // just llvm bitcode. In that case write bitcode, and possibly
+    // delete the bitcode if it wasn't requested. Don't generate the
+    // machine code, instead copy the .o file from the .bc
+    let write_bc = config.emit_bc || config.obj_is_bitcode;
+    let rm_bc = !config.emit_bc && config.obj_is_bitcode;
+    let write_obj = config.emit_obj && !config.obj_is_bitcode;
+    let copy_bc_to_obj = config.emit_obj && config.obj_is_bitcode;
+
+    let bc_out = output_names.with_extension(&format!("{}.bc", name_extra));
+    let obj_out = output_names.with_extension(&format!("{}.o", name_extra));
+
+    if write_bc {
+        let bc_out_c = path2cstr(&bc_out);
+        llvm::LLVMWriteBitcodeToFile(llmod, bc_out_c.as_ptr());
     }
 
     time(config.time_passes, &format!("codegen passes [{}]", cgcx.worker), || {
@@ -568,14 +583,27 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext,
             }
         }
 
-        if config.emit_obj {
-            let path = output_names.with_extension(&format!("{}.o", name_extra));
+        if write_obj {
             with_codegen(tm, llmod, config.no_builtins, |cpm| {
-                write_output_file(cgcx.handler, tm, cpm, llmod, &path, llvm::ObjectFileType);
+                write_output_file(cgcx.handler, tm, cpm, llmod, &obj_out, llvm::ObjectFileType);
             });
         }
     });
 
+    if copy_bc_to_obj {
+        debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
+        if let Err(e) = fs::copy(&bc_out, &obj_out) {
+            cgcx.handler.err(&format!("failed to copy bitcode to object file: {}", e));
+        }
+    }
+
+    if rm_bc {
+        debug!("removing_bitcode {:?}", bc_out);
+        if let Err(e) = fs::remove_file(&bc_out) {
+            cgcx.handler.err(&format!("failed to remove bitcode: {}", e));
+        }
+    }
+
     llvm::LLVMDisposeModule(llmod);
     llvm::LLVMContextDispose(llcx);
     llvm::LLVMRustDisposeTargetMachine(tm);