Auto merge of #24177 - alexcrichton:rustdoc, r=aturon

This commit series starts out with more official test harness support for rustdoc tests, and then each commit afterwards adds a test (where appropriate). Each commit should also test and finish independently of all others (they're all pretty separable).

I've uploaded a [copy of the documentation](http://people.mozilla.org/~acrichton/doc/std/) generated after all these commits were applied, and a double check on issues being closed would be greatly appreciated! I'll also browse the docs a bit and make sure nothing regressed too horribly.

configure

@@ -1128,6 +1128,7 @@ do
     make_dir $h/test/debuginfo-gdb
     make_dir $h/test/debuginfo-lldb
     make_dir $h/test/codegen
+    make_dir $h/test/rustdoc
 done
 
 # Configure submodules

mk/crates.mk

@@ -118,42 +118,13 @@ ONLY_RLIB_collections := 1
 ONLY_RLIB_unicode := 1
 ONLY_RLIB_rustc_bitflags := 1
 
+# Documented-by-default crates
+DOC_CRATES := std alloc collections core libc unicode
+
 ################################################################################
 # You should not need to edit below this line
 ################################################################################
 
-# On channels where the only usable crate is std, only build documentation for
-# std. This keeps distributions small and doesn't clutter up the API docs with
-# confusing internal details from the crates behind the facade.
-#
-# (Disabled while cmr figures out how to change rustdoc to make reexports work
-# slightly nicer. Otherwise, all cross-crate links to Vec will go to
-# libcollections, breaking them, and [src] links for anything reexported will
-# not work.)
-
-#ifeq ($(CFG_RELEASE_CHANNEL),stable)
-#DOC_CRATES := std
-#else
-#ifeq ($(CFG_RELEASE_CHANNEL),beta)
-#DOC_CRATES := std
-#else
-DOC_CRATES := $(filter-out rustc, \
-	   $(filter-out rustc_trans, \
-	   $(filter-out rustc_typeck, \
-	   $(filter-out rustc_borrowck, \
-	   $(filter-out rustc_resolve, \
-	   $(filter-out rustc_driver, \
-	   $(filter-out rustc_privacy, \
-	   $(filter-out rustc_lint, \
-	   $(filter-out log, \
-	   $(filter-out getopts, \
-	   $(filter-out syntax, $(CRATES))))))))))))
-#endif
-#endif
-COMPILER_DOC_CRATES := rustc rustc_trans rustc_borrowck rustc_resolve \
-                       rustc_typeck rustc_driver syntax rustc_privacy \
-                       rustc_lint
-
 # This macro creates some simple definitions for each crate being built, just
 # some munging of all of the parameters above.
 #

mk/docs.mk

@@ -250,18 +250,20 @@ endif
 doc/$(1)/:
 	$$(Q)mkdir -p $$@
 
-$(2) += doc/$(1)/index.html
 doc/$(1)/index.html: CFG_COMPILER_HOST_TRIPLE = $(CFG_TARGET)
 doc/$(1)/index.html: $$(LIB_DOC_DEP_$(1)) doc/$(1)/
 	@$$(call E, rustdoc: $$@)
 	$$(Q)CFG_LLVM_LINKAGE_FILE=$$(LLVM_LINKAGE_PATH_$(CFG_BUILD)) \
-		$$(RUSTDOC) --cfg dox --cfg stage2 $$<
+		$$(RUSTDOC) --cfg dox --cfg stage2 $$(RUSTFLAGS_$(1)) $$<
 endef
 
-$(foreach crate,$(DOC_CRATES),$(eval $(call DEF_LIB_DOC,$(crate),DOC_TARGETS)))
+$(foreach crate,$(CRATES),$(eval $(call DEF_LIB_DOC,$(crate))))
 
+COMPILER_DOC_TARGETS := $(CRATES:%=doc/%/index.html)
 ifdef CFG_COMPILER_DOCS
-  $(foreach crate,$(COMPILER_DOC_CRATES),$(eval $(call DEF_LIB_DOC,$(crate),COMPILER_DOC_TARGETS)))
+  DOC_TARGETS += $(COMPILER_DOC_TARGETS)
+else
+  DOC_TARGETS += $(DOC_CRATES:%=doc/%/index.html)
 endif
 
 ifdef CFG_DISABLE_DOCS

mk/tests.mk

@@ -22,9 +22,11 @@ $(eval $(call RUST_CRATE,coretest))
 DEPS_collectionstest :=
 $(eval $(call RUST_CRATE,collectionstest))
 
-TEST_TARGET_CRATES = $(filter-out core unicode,$(TARGET_CRATES)) collectionstest coretest
+TEST_TARGET_CRATES = $(filter-out core unicode,$(TARGET_CRATES)) \
+			collectionstest coretest
 TEST_DOC_CRATES = $(DOC_CRATES)
-TEST_HOST_CRATES = $(filter-out rustc_typeck rustc_borrowck rustc_resolve rustc_trans rustc_lint,\
+TEST_HOST_CRATES = $(filter-out rustc_typeck rustc_borrowck rustc_resolve \
+		   		rustc_trans rustc_lint,\
                      $(HOST_CRATES))
 TEST_CRATES = $(TEST_TARGET_CRATES) $(TEST_HOST_CRATES)
 
@@ -304,6 +306,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
     check-stage$(1)-T-$(2)-H-$(3)-rpass-full-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-cfail-full-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-rmake-exec \
+	check-stage$(1)-T-$(2)-H-$(3)-rustdocck-exec \
         check-stage$(1)-T-$(2)-H-$(3)-crates-exec \
         check-stage$(1)-T-$(2)-H-$(3)-doc-crates-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-bench-exec \
@@ -471,6 +474,7 @@ DEBUGINFO_GDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
 DEBUGINFO_LLDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
 CODEGEN_RS := $(wildcard $(S)src/test/codegen/*.rs)
 CODEGEN_CC := $(wildcard $(S)src/test/codegen/*.cc)
+RUSTDOCCK_RS := $(wildcard $(S)src/test/rustdoc/*.rs)
 
 # perf tests are the same as bench tests only they run under
 # a performance monitor.
@@ -489,6 +493,7 @@ PRETTY_TESTS := $(PRETTY_RS)
 DEBUGINFO_GDB_TESTS := $(DEBUGINFO_GDB_RS)
 DEBUGINFO_LLDB_TESTS := $(DEBUGINFO_LLDB_RS)
 CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
+RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)
 
 CTEST_SRC_BASE_rpass = run-pass
 CTEST_BUILD_BASE_rpass = run-pass
@@ -550,6 +555,11 @@ CTEST_BUILD_BASE_codegen = codegen
 CTEST_MODE_codegen = codegen
 CTEST_RUNTOOL_codegen = $(CTEST_RUNTOOL)
 
+CTEST_SRC_BASE_rustdocck = rustdoc
+CTEST_BUILD_BASE_rustdocck = rustdoc
+CTEST_MODE_rustdocck = rustdoc
+CTEST_RUNTOOL_rustdocck = $(CTEST_RUNTOOL)
+
 # CTEST_DISABLE_$(TEST_GROUP), if set, will cause the test group to be
 # disabled and the associated message to be printed as a warning
 # during attempts to run those tests.
@@ -618,12 +628,14 @@ CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) := \
 		--compile-lib-path $$(HLIB$(1)_H_$(3)) \
         --run-lib-path $$(TLIB$(1)_T_$(2)_H_$(3)) \
         --rustc-path $$(HBIN$(1)_H_$(3))/rustc$$(X_$(3)) \
+        --rustdoc-path $$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
         --clang-path $(if $(CFG_CLANG),$(CFG_CLANG),clang) \
         --llvm-bin-path $(CFG_LLVM_INST_DIR_$(CFG_BUILD))/bin \
         --aux-base $$(S)src/test/auxiliary/ \
         --stage-id stage$(1)-$(2) \
         --target $(2) \
         --host $(3) \
+	--python $$(CFG_PYTHON) \
         --gdb-version="$(CFG_GDB_VERSION)" \
         --lldb-version="$(CFG_LLDB_VERSION)" \
         --android-cross-path=$(CFG_ANDROID_CROSS_PATH) \
@@ -660,6 +672,9 @@ CTEST_DEPS_debuginfo-lldb_$(1)-T-$(2)-H-$(3) = $$(DEBUGINFO_LLDB_TESTS) \
                                                $(S)src/etc/lldb_batchmode.py \
                                                $(S)src/etc/lldb_rust_formatters.py
 CTEST_DEPS_codegen_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_TESTS)
+CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
+        $$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
+	$(S)src/etc/htmldocck.py
 
 endef
 
@@ -722,7 +737,8 @@ endif
 
 endef
 
-CTEST_NAMES = rpass rpass-valgrind rpass-full cfail-full rfail cfail pfail bench perf debuginfo-gdb debuginfo-lldb codegen
+CTEST_NAMES = rpass rpass-valgrind rpass-full cfail-full rfail cfail pfail \
+	bench perf debuginfo-gdb debuginfo-lldb codegen rustdocck
 
 $(foreach host,$(CFG_HOST), \
  $(eval $(foreach target,$(CFG_TARGET), \
@@ -890,6 +906,7 @@ TEST_GROUPS = \
 	bench \
 	perf \
 	rmake \
+	rustdocck \
 	debuginfo-gdb \
 	debuginfo-lldb \
 	codegen \

src/compiletest/common.rs

@@ -23,7 +23,8 @@ pub enum Mode {
     Pretty,
     DebugInfoGdb,
     DebugInfoLldb,
-    Codegen
+    Codegen,
+    Rustdoc,
 }
 
 impl FromStr for Mode {
@@ -39,6 +40,7 @@ impl FromStr for Mode {
           "debuginfo-lldb" => Ok(DebugInfoLldb),
           "debuginfo-gdb" => Ok(DebugInfoGdb),
           "codegen" => Ok(Codegen),
+          "rustdoc" => Ok(Rustdoc),
           _ => Err(()),
         }
     }
@@ -56,6 +58,7 @@ impl fmt::Display for Mode {
             DebugInfoGdb => "debuginfo-gdb",
             DebugInfoLldb => "debuginfo-lldb",
             Codegen => "codegen",
+            Rustdoc => "rustdoc",
         }, f)
     }
 }
@@ -71,6 +74,12 @@ pub struct Config {
     // The rustc executable
     pub rustc_path: PathBuf,
 
+    // The rustdoc executable
+    pub rustdoc_path: PathBuf,
+
+    // The python executable
+    pub python: String,
+
     // The clang executable
     pub clang_path: Option<PathBuf>,
 

src/compiletest/compiletest.rs

@@ -60,6 +60,8 @@ pub fn parse_config(args: Vec<String> ) -> Config {
         vec!(reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
           reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
           reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
+          reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
+          reqopt("", "python", "path to python to use for doc tests", "PATH"),
           optopt("", "clang-path", "path to  executable for codegen tests", "PATH"),
           optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
           optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
@@ -128,6 +130,8 @@ pub fn parse_config(args: Vec<String> ) -> Config {
         compile_lib_path: matches.opt_str("compile-lib-path").unwrap(),
         run_lib_path: matches.opt_str("run-lib-path").unwrap(),
         rustc_path: opt_path(matches, "rustc-path"),
+        rustdoc_path: opt_path(matches, "rustdoc-path"),
+        python: matches.opt_str("python").unwrap(),
         clang_path: matches.opt_str("clang-path").map(|s| PathBuf::from(&s)),
         valgrind_path: matches.opt_str("valgrind-path"),
         force_valgrind: matches.opt_present("force-valgrind"),
@@ -168,6 +172,7 @@ pub fn log_config(config: &Config) {
     logv(c, format!("compile_lib_path: {:?}", config.compile_lib_path));
     logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
     logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
+    logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path.display()));
     logv(c, format!("src_base: {:?}", config.src_base.display()));
     logv(c, format!("build_base: {:?}", config.build_base.display()));
     logv(c, format!("stage_id: {}", config.stage_id));