Merge pull request #62151 from thockin/build-cleanup-with-go-1-10

Automatic merge from submit-queue (batch tested with PRs 62495, 63003, 62829, 62151, 62002). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Build cleanup with go 1.10

Simplify our build/test scripts now that go 1.10 is in play.  It has caching that seems to actually work.

**Release note**:
```release-note
NONE
```

build/root/Makefile.generated_files

@@ -35,7 +35,7 @@ SHELL := /bin/bash
 # This rule collects all the generated file sets into a single rule.  Other
 # rules should depend on this to ensure generated files are rebuilt.
 .PHONY: generated_files
-generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi
+generated_files: gen_deepcopy gen_defaulter gen_conversion gen_openapi gen_bindata
 
 .PHONY: verify_generated_files
 verify_generated_files: verify_gen_deepcopy    \
@@ -486,110 +486,6 @@ $(DEFAULTER_GEN):
 	hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/defaulter-gen
 	touch $@
 
-#
-# Open-api generation
-#
-# Any package that wants open-api functions generated must include a
-# comment-tag in column 0 of one file of the form:
-#     // +k8s:openapi-gen=true
-#
-# The result file, in each pkg, of open-api generation.
-OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
-OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
-OPENAPI_OUTPUT_PKG := pkg/generated/openapi
-
-# The tool used to generate open apis.
-OPENAPI_GEN := $(BIN_DIR)/openapi-gen
-
-# Find all the directories that request open-api generation.
-ifeq ($(DBG_MAKEFILE),1)
-    $(warning ***** finding all +k8s:openapi-gen tags)
-endif
-OPENAPI_DIRS := $(shell                                             \
-    grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES)  \
-        | xargs -n1 dirname                                         \
-        | LC_ALL=C sort -u                                          \
-)
-
-OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
-
-# This rule is the user-friendly entrypoint for openapi generation.
-.PHONY: gen_openapi
-gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
-
-# For each dir in OPENAPI_DIRS, this establishes a dependency between the
-# output file and the input files that should trigger a rebuild.
-#
-# Note that this is a deps-only statement, not a full rule (see below).  This
-# has to be done in a distinct step because wildcards don't work in static
-# pattern rules.
-#
-# The '$(eval)' is needed because this has a different RHS for each LHS, and
-# would otherwise produce results that make can't parse.
-#
-# We depend on the $(GOFILES_META).stamp to detect when the set of input files
-# has changed.  This allows us to detect deleted input files.
-$(foreach dir, $(OPENAPI_DIRS), $(eval                                     \
-    $(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp           \
-                                 $(gofiles__$(dir))                        \
-))
-
-# How to regenerate open-api code.  This emits a single file for all results.
-$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN)
-	function run_gen_openapi() {                                                        \
-	    ./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
-	        --v $(KUBE_VERBOSE)                                                         \
-	        --logtostderr                                                               \
-	        -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g')  \
-	        -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG)                                    \
-	        -O $(OPENAPI_BASENAME)                                                      \
-	        "$$@";                                                                      \
-	};                                                                                  \
-	run_gen_openapi
-
-# This calculates the dependencies for the generator tool, so we only rebuild
-# it when needed.  It is PHONY so that it always runs, but it only updates the
-# file if the contents have actually changed.  We 'sinclude' this later.
-.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk
-$(META_DIR)/$(OPENAPI_GEN).mk:
-	mkdir -p $(@D);                                                       \
-	(echo -n "$(OPENAPI_GEN): ";                                          \
-	 ./hack/run-in-gopath.sh go list                                      \
-	     -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}'  \
-	     ./vendor/k8s.io/code-generator/cmd/openapi-gen                   \
-	     | grep --color=never "^$(PRJ_SRC_PATH)/"                         \
-	     | xargs ./hack/run-in-gopath.sh go list                          \
-	         -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}'  \
-	     | paste -sd' ' -                                                 \
-	     | sed 's/ / \\=,/g'                                              \
-	     | tr '=,' '\n\t'                                                 \
-	     | sed "s|$$(pwd -P)/||";                                         \
-	) > $@.tmp;                                                           \
-	if ! cmp -s $@.tmp $@; then                                           \
-	    if [[ "$(DBG_CODEGEN)" == 1 ]]; then                              \
-	        echo "DBG: $(OPENAPI_GEN).mk changed";                        \
-	    fi;                                                               \
-	    cat $@.tmp > $@;                                                  \
-	    rm -f $@.tmp;                                                     \
-	fi
-
-# Include dependency info for the generator tool.  This will cause the rule of
-# the same name to be considered and if it is updated, make will restart.
-sinclude $(META_DIR)/$(OPENAPI_GEN).mk
-
-# How to build the generator tool.  The deps for this are defined in
-# the $(OPENAPI_GEN).mk, above.
-#
-# A word on the need to touch: This rule might trigger if, for example, a
-# non-Go file was added or deleted from a directory on which this depends.
-# This target needs to be reconsidered, but Go realizes it doesn't actually
-# have to be rebuilt.  In that case, make will forever see the dependency as
-# newer than the binary, and try to rebuild it over and over.  So we touch it,
-# and make is happy.
-$(OPENAPI_GEN):
-	hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen
-	touch $@
-
 #
 # Conversion generation
 #
@@ -805,3 +701,168 @@ sinclude $(META_DIR)/$(CONVERSION_GEN).mk
 $(CONVERSION_GEN):
 	hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/conversion-gen
 	touch $@
+
+#
+# Open-api generation
+#
+# Any package that wants open-api functions generated must include a
+# comment-tag in column 0 of one file of the form:
+#     // +k8s:openapi-gen=true
+#
+# The result file, in each pkg, of open-api generation.
+OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
+OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
+OPENAPI_OUTPUT_PKG := pkg/generated/openapi
+
+# The tool used to generate open apis.
+OPENAPI_GEN := $(BIN_DIR)/openapi-gen
+
+# Find all the directories that request open-api generation.
+ifeq ($(DBG_MAKEFILE),1)
+    $(warning ***** finding all +k8s:openapi-gen tags)
+endif
+OPENAPI_DIRS := $(shell                                             \
+    grep --color=never -l '+k8s:openapi-gen=' $(ALL_K8S_TAG_FILES)  \
+        | xargs -n1 dirname                                         \
+        | LC_ALL=C sort -u                                          \
+)
+
+OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
+
+# This rule is the user-friendly entrypoint for openapi generation.
+.PHONY: gen_openapi
+gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
+
+# For each dir in OPENAPI_DIRS, this establishes a dependency between the
+# output file and the input files that should trigger a rebuild.
+#
+# Note that this is a deps-only statement, not a full rule (see below).  This
+# has to be done in a distinct step because wildcards don't work in static
+# pattern rules.
+#
+# The '$(eval)' is needed because this has a different RHS for each LHS, and
+# would otherwise produce results that make can't parse.
+#
+# We depend on the $(GOFILES_META).stamp to detect when the set of input files
+# has changed.  This allows us to detect deleted input files.
+$(foreach dir, $(OPENAPI_DIRS), $(eval                                     \
+    $(OPENAPI_OUTFILE): $(META_DIR)/$(dir)/$(GOFILES_META).stamp           \
+                                 $(gofiles__$(dir))                        \
+))
+
+# How to regenerate open-api code.  This emits a single file for all results.
+$(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(OPENAPI_GEN)
+	function run_gen_openapi() {                                                        \
+	    ./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
+	        --v $(KUBE_VERBOSE)                                                         \
+	        --logtostderr                                                               \
+	        -i $$(echo $(addprefix $(PRJ_SRC_PATH)/, $(OPENAPI_DIRS)) | sed 's/ /,/g')  \
+	        -p $(PRJ_SRC_PATH)/$(OPENAPI_OUTPUT_PKG)                                    \
+	        -O $(OPENAPI_BASENAME)                                                      \
+	        "$$@";                                                                      \
+	};                                                                                  \
+	run_gen_openapi
+
+# This calculates the dependencies for the generator tool, so we only rebuild
+# it when needed.  It is PHONY so that it always runs, but it only updates the
+# file if the contents have actually changed.  We 'sinclude' this later.
+.PHONY: $(META_DIR)/$(OPENAPI_GEN).mk
+$(META_DIR)/$(OPENAPI_GEN).mk:
+	mkdir -p $(@D);                                                       \
+	(echo -n "$(OPENAPI_GEN): ";                                          \
+	 ./hack/run-in-gopath.sh go list                                      \
+	     -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}'  \
+	     ./vendor/k8s.io/code-generator/cmd/openapi-gen                   \
+	     | grep --color=never "^$(PRJ_SRC_PATH)/"                         \
+	     | xargs ./hack/run-in-gopath.sh go list                          \
+	         -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}'  \
+	     | paste -sd' ' -                                                 \
+	     | sed 's/ / \\=,/g'                                              \
+	     | tr '=,' '\n\t'                                                 \
+	     | sed "s|$$(pwd -P)/||";                                         \
+	) > $@.tmp;                                                           \
+	if ! cmp -s $@.tmp $@; then                                           \
+	    if [[ "$(DBG_CODEGEN)" == 1 ]]; then                              \
+	        echo "DBG: $(OPENAPI_GEN).mk changed";                        \
+	    fi;                                                               \
+	    cat $@.tmp > $@;                                                  \
+	    rm -f $@.tmp;                                                     \
+	fi
+
+# Include dependency info for the generator tool.  This will cause the rule of
+# the same name to be considered and if it is updated, make will restart.
+sinclude $(META_DIR)/$(OPENAPI_GEN).mk
+
+# How to build the generator tool.  The deps for this are defined in
+# the $(OPENAPI_GEN).mk, above.
+#
+# A word on the need to touch: This rule might trigger if, for example, a
+# non-Go file was added or deleted from a directory on which this depends.
+# This target needs to be reconsidered, but Go realizes it doesn't actually
+# have to be rebuilt.  In that case, make will forever see the dependency as
+# newer than the binary, and try to rebuild it over and over.  So we touch it,
+# and make is happy.
+$(OPENAPI_GEN):
+	hack/make-rules/build.sh ./vendor/k8s.io/code-generator/cmd/openapi-gen
+	touch $@
+
+#
+# bindata generation
+#
+
+# The tool used to generate bindata files.
+BINDATA_GEN := $(BIN_DIR)/go-bindata
+
+# A wrapper script that generates all bindata files.  It is fast enough that we
+# don't care.
+BINDATA_SCRIPT := hack/generate-bindata.sh
+
+# This rule is the user-friendly entrypoint for bindata generation.
+.PHONY: gen_bindata
+gen_bindata: $(BINDATA_GEN) FORCE
+	./hack/run-in-gopath.sh $(BINDATA_SCRIPT)
+
+FORCE:
+
+# This calculates the dependencies for the generator tool, so we only rebuild
+# it when needed.  It is PHONY so that it always runs, but it only updates the
+# file if the contents have actually changed.  We 'sinclude' this later.
+.PHONY: $(META_DIR)/$(BINDATA_GEN).mk
+$(META_DIR)/$(BINDATA_GEN).mk:
+	mkdir -p $(@D);                                                       \
+	(echo -n "$(BINDATA_GEN): ";                                          \
+	 ./hack/run-in-gopath.sh go list                                      \
+	     -f '{{.ImportPath}}{{"\n"}}{{range .Deps}}{{.}}{{"\n"}}{{end}}'  \
+	     ./vendor/github.com/jteeuwen/go-bindata/go-bindata               \
+	     | grep --color=never "^$(PRJ_SRC_PATH)/"                         \
+	     | xargs ./hack/run-in-gopath.sh go list                          \
+	         -f '{{$$d := .Dir}}{{$$d}}{{"\n"}}{{range .GoFiles}}{{$$d}}/{{.}}{{"\n"}}{{end}}'  \
+	     | paste -sd' ' -                                                 \
+	     | sed 's/ / \\=,/g'                                              \
+	     | tr '=,' '\n\t'                                                 \
+	     | sed "s|$$(pwd -P)/||";                                         \
+	) > $@.tmp;                                                           \
+	if ! cmp -s $@.tmp $@; then                                           \
+	    if [[ "$(DBG_CODEGEN)" == 1 ]]; then                              \
+	        echo "DBG: $(BINDATA_GEN).mk changed";                        \
+	    fi;                                                               \
+	    cat $@.tmp > $@;                                                  \
+	    rm -f $@.tmp;                                                     \
+	fi
+
+# Include dependency info for the generator tool.  This will cause the rule of
+# the same name to be considered and if it is updated, make will restart.
+sinclude $(META_DIR)/$(BINDATA_GEN).mk
+
+# How to build the generator tool.  The deps for this are defined in
+# the $(BINDATA_GEN).mk, above.
+#
+# A word on the need to touch: This rule might trigger if, for example, a
+# non-Go file was added or deleted from a directory on which this depends.
+# This target needs to be reconsidered, but Go realizes it doesn't actually
+# have to be rebuilt.  In that case, make will forever see the dependency as
+# newer than the binary, and try to rebuild it over and over.  So we touch it,
+# and make is happy.
+$(BINDATA_GEN):
+	hack/make-rules/build.sh ./vendor/github.com/jteeuwen/go-bindata/go-bindata
+	touch $@

hack/BUILD

@@ -19,7 +19,6 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//hack/boilerplate:all-srcs",
-        "//hack/cmd/teststale:all-srcs",
         "//hack/e2e-internal:all-srcs",
         "//hack/lib:all-srcs",
         "//hack/make-rules:all-srcs",

hack/cmd/teststale/BUILD

@@ -1,39 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_binary",
-    "go_library",
-    "go_test",
-)
-
-go_binary(
-    name = "teststale",
-    embed = [":go_default_library"],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["teststale_test.go"],
-    embed = [":go_default_library"],
-)
-
-go_library(
-    name = "go_default_library",
-    srcs = ["teststale.go"],
-    importpath = "k8s.io/kubernetes/hack/cmd/teststale",
-    deps = ["//vendor/github.com/golang/glog:go_default_library"],
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-)

hack/cmd/teststale/teststale.go

@@ -1,209 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// teststale checks the staleness of a test binary. go test -c builds a test
-// binary but it does no staleness check. In other words, every time one runs
-// go test -c, it compiles the test packages and links the binary even when
-// nothing has changed. This program helps to mitigate that problem by allowing
-// to check the staleness of a given test package and its binary.
-package main
-
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"time"
-
-	"github.com/golang/glog"
-)
-
-const usageHelp = "" +
-	`This program checks the staleness of a given test package and its test
-binary so that one can make a decision about re-building the test binary.
-
-Usage:
-  teststale -binary=/path/to/test/binary -package=package
-
-Example:
-  teststale -binary="$HOME/gosrc/bin/e2e.test" -package="k8s.io/kubernetes/test/e2e"
-
-`
-
-var (
-	binary  = flag.String("binary", "", "filesystem path to the test binary file. Example: \"$HOME/gosrc/bin/e2e.test\"")
-	pkgPath = flag.String("package", "", "import path of the test package in the format used while importing packages. Example: \"k8s.io/kubernetes/test/e2e\"")
-)
-
-func usage() {
-	fmt.Fprintln(os.Stderr, usageHelp)
-	fmt.Fprintln(os.Stderr, "Flags:")
-	flag.PrintDefaults()
-	os.Exit(2)
-}
-
-// golist is an interface emulating the `go list` command to get package information.
-// TODO: Evaluate using `go/build` package instead. It doesn't provide staleness
-// information, but we can probably run `go list` and `go/build.Import()` concurrently
-// in goroutines and merge the results. Evaluate if that's faster.
-type golist interface {
-	pkgInfo(pkgPaths []string) ([]pkg, error)
-}
-
-// execmd implements the `golist` interface.
-type execcmd struct {
-	cmd  string
-	args []string
-	env  []string
-}
-
-func (e *execcmd) pkgInfo(pkgPaths []string) ([]pkg, error) {
-	args := append(e.args, pkgPaths...)
-	cmd := exec.Command(e.cmd, args...)
-	cmd.Env = e.env
-
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, fmt.Errorf("failed to obtain the metadata output stream: %v", err)
-	}
-
-	dec := json.NewDecoder(stdout)
-
-	// Start executing the command
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("command did not start: %v", err)
-	}
-
-	var pkgs []pkg
-	for {
-		var p pkg
-		if err := dec.Decode(&p); err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, fmt.Errorf("failed to unmarshal metadata for package %s: %v", p.ImportPath, err)
-		}
-		pkgs = append(pkgs, p)
-	}
-
-	if err := cmd.Wait(); err != nil {
-		return nil, fmt.Errorf("command did not complete: %v", err)
-	}
-	return pkgs, nil
-}
-
-type pkg struct {
-	Dir          string
-	ImportPath   string
-	Target       string
-	Stale        bool
-	TestGoFiles  []string
-	TestImports  []string
-	XTestGoFiles []string
-	XTestImports []string
-}
-
-func (p *pkg) isNewerThan(cmd golist, buildTime time.Time) bool {
-	// If the package itself is stale, then we have to rebuild the whole thing anyway.
-	if p.Stale {
-		return true
-	}
-
-	// Test for file staleness
-	for _, f := range p.TestGoFiles {
-		if isNewerThan(filepath.Join(p.Dir, f), buildTime) {
-			glog.V(4).Infof("test Go file %s is stale", f)
-			return true
-		}
-	}
-	for _, f := range p.XTestGoFiles {
-		if isNewerThan(filepath.Join(p.Dir, f), buildTime) {
-			glog.V(4).Infof("external test Go file %s is stale", f)
-			return true
-		}
-	}
-
-	imps := []string{}
-	imps = append(imps, p.TestImports...)
-	imps = append(imps, p.XTestImports...)
-
-	// This calls `go list` the second time. This is required because the first
-	// call to `go list` checks the staleness of the package in question by
-	// looking the non-test dependencies, but it doesn't look at the test
-	// dependencies. However, it returns the list of test dependencies. This
-	// second call to `go list` checks the staleness of all the test
-	// dependencies.
-	pkgs, err := cmd.pkgInfo(imps)
-	if err != nil || len(pkgs) < 1 {
-		glog.V(4).Infof("failed to obtain metadata for packages %s: %v", imps, err)
-		return true
-	}
-
-	for _, p := range pkgs {
-		if p.Stale {
-			glog.V(4).Infof("import %q is stale", p.ImportPath)
-			return true
-		}
-	}
-
-	return false
-}
-
-func isNewerThan(filename string, buildTime time.Time) bool {
-	stat, err := os.Stat(filename)
-	if err != nil {
-		return true
-	}
-	return stat.ModTime().After(buildTime)
-}
-
-// isTestStale checks if the test binary is stale and needs to rebuilt.
-// Some of the ideas here are inspired by how Go does staleness checks.
-func isTestStale(cmd golist, binPath, pkgPath string) bool {
-	bStat, err := os.Stat(binPath)
-	if err != nil {
-		glog.V(4).Infof("Couldn't obtain the modified time of the binary %s: %v", binPath, err)
-		return true
-	}
-	buildTime := bStat.ModTime()
-
-	pkgs, err := cmd.pkgInfo([]string{pkgPath})
-	if err != nil || len(pkgs) < 1 {
-		glog.V(4).Infof("Couldn't retrieve test package information for package %s: %v", pkgPath, err)
-		return false
-	}
-
-	return pkgs[0].isNewerThan(cmd, buildTime)
-}
-
-func main() {
-	flag.Usage = usage
-	flag.Parse()
-
-	cmd := &execcmd{
-		cmd: "go",
-		args: []string{
-			"list",
-			"-json",
-		},
-		env: os.Environ(),
-	}
-	if !isTestStale(cmd, *binary, *pkgPath) {
-		os.Exit(1)
-	}
-}

hack/cmd/teststale/teststale_test.go

@@ -1,325 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path"
-	"path/filepath"
-	"testing"
-	"time"
-)
-
-const (
-	// seed for rand.Source to generate data for files
-	seed int64 = 42
-
-	// 1K binary file
-	binLen = 1024
-
-	// Directory of the test package relative to $GOPATH
-	testImportDir = "example.com/proj/pkg"
-)
-
-var (
-	pastHour = time.Now().Add(-1 * time.Hour)
-
-	// The test package we are testing against
-	testPkg = path.Join(testImportDir, "test")
-)
-
-// fakegolist implements the `golist` interface providing fake package information for testing.
-type fakegolist struct {
-	dir       string
-	importMap map[string]pkg
-	testFiles []string
-	binfile   string
-}
-
-func newFakegolist() (*fakegolist, error) {
-	dir, err := ioutil.TempDir("", "teststale")
-	if err != nil {
-		// test can't proceed without a temp directory.
-		return nil, fmt.Errorf("failed to create a temp directory for testing: %v", err)
-	}
-
-	// Set the temp directory as the $GOPATH
-	if err := os.Setenv("GOPATH", dir); err != nil {
-		// can't proceed without pointing the $GOPATH to the temp directory.
-		return nil, fmt.Errorf("failed to set \"$GOPATH\" pointing to %q: %v", dir, err)
-	}
-
-	// Setup $GOPATH directory layout.
-	// Yeah! I am bored of repeatedly writing "if err != nil {}"!
-	if os.MkdirAll(filepath.Join(dir, "bin"), 0750) != nil ||
-		os.MkdirAll(filepath.Join(dir, "pkg", "linux_amd64"), 0750) != nil ||
-		os.MkdirAll(filepath.Join(dir, "src"), 0750) != nil {
-		return nil, fmt.Errorf("failed to setup the $GOPATH directory structure")
-	}
-
-	// Create a temp file to represent the test binary.
-	binfile, err := ioutil.TempFile("", "testbin")
-	if err != nil {
-		return nil, fmt.Errorf("failed to create the temp file to represent the test binary: %v", err)
-	}
-
-	// Could have used crypto/rand instead, but it doesn't matter.
-	rr := rand.New(rand.NewSource(42))
-	bin := make([]byte, binLen)
-	if _, err = rr.Read(bin); err != nil {
-		return nil, fmt.Errorf("couldn't read from the random source: %v", err)
-	}
-	if _, err := binfile.Write(bin); err != nil {
-		return nil, fmt.Errorf("couldn't write to the binary file %q: %v", binfile.Name(), err)
-	}
-	if err := binfile.Close(); err != nil {
-		// It is arguable whether this should be fatal.
-		return nil, fmt.Errorf("failed to close the binary file %q: %v", binfile.Name(), err)
-	}
-
-	if err := os.Chtimes(binfile.Name(), time.Now(), time.Now()); err != nil {
-		return nil, fmt.Errorf("failed to modify the mtime of the binary file %q: %v", binfile.Name(), err)
-	}
-
-	// Create test source files directory.
-	testdir := filepath.Join(dir, "src", testPkg)
-	if err := os.MkdirAll(testdir, 0750); err != nil {
-		return nil, fmt.Errorf("failed to create test source directory %q: %v", testdir, err)
-	}
-
-	fgl := &fakegolist{
-		dir: dir,
-		importMap: map[string]pkg{
-			"example.com/proj/pkg/test": {
-				Dir:        path.Join(dir, "src", testPkg),
-				ImportPath: testPkg,
-				Target:     path.Join(dir, "pkg", "linux_amd64", testImportDir, "test.a"),
-				Stale:      false,
-				TestGoFiles: []string{
-					"foo_test.go",
-					"bar_test.go",
-				},
-				TestImports: []string{
-					"example.com/proj/pkg/p1",
-					"example.com/proj/pkg/p1/c11",
-					"example.com/proj/pkg/p2",
-					"example.com/proj/cmd/p3/c12/c23",
-					"strings",
-					"testing",
-				},
-				XTestGoFiles: []string{
-					"xfoo_test.go",
-					"xbar_test.go",
-					"xbaz_test.go",
-				},
-				XTestImports: []string{
-					"example.com/proj/pkg/test",
-					"example.com/proj/pkg/p1",
-					"example.com/proj/cmd/p3/c12/c23",
-					"os",
-					"testing",
-				},
-			},
-			"example.com/proj/pkg/p1":         {Stale: false},
-			"example.com/proj/pkg/p1/c11":     {Stale: false},
-			"example.com/proj/pkg/p2":         {Stale: false},
-			"example.com/proj/cmd/p3/c12/c23": {Stale: false},
-			"strings":                         {Stale: false},
-			"testing":                         {Stale: false},
-			"os":                              {Stale: false},
-		},
-		testFiles: []string{
-			"foo_test.go",
-			"bar_test.go",
-			"xfoo_test.go",
-			"xbar_test.go",
-			"xbaz_test.go",
-		},
-		binfile: binfile.Name(),
-	}
-
-	// Create test source files.
-	for _, fn := range fgl.testFiles {
-		fp := filepath.Join(testdir, fn)
-		if _, err := os.Create(fp); err != nil {
-			return nil, fmt.Errorf("failed to create the test file %q: %v", fp, err)
-		}
-		if err := os.Chtimes(fp, time.Now(), pastHour); err != nil {
-			return nil, fmt.Errorf("failed to modify the mtime of the test file %q: %v", binfile.Name(), err)
-		}
-	}
-
-	return fgl, nil
-}
-
-func (fgl *fakegolist) pkgInfo(pkgPaths []string) ([]pkg, error) {
-	var pkgs []pkg
-	for _, path := range pkgPaths {
-		p, ok := fgl.importMap[path]
-		if !ok {
-			return nil, fmt.Errorf("package %q not found", path)
-		}
-		pkgs = append(pkgs, p)
-	}
-	return pkgs, nil
-}
-
-func (fgl *fakegolist) chMtime(filename string, mtime time.Time) error {
-	for _, fn := range fgl.testFiles {
-		if fn == filename {
-			fp := filepath.Join(fgl.dir, "src", testPkg, fn)
-			if err := os.Chtimes(fp, time.Now(), mtime); err != nil {
-				return fmt.Errorf("failed to modify the mtime of %q: %v", filename, err)
-			}
-			return nil
-		}
-	}
-	return fmt.Errorf("file %q not found", filename)
-}
-
-func (fgl *fakegolist) chStale(pkg string, stale bool) error {
-	if p, ok := fgl.importMap[pkg]; ok {
-		p.Stale = stale
-		fgl.importMap[pkg] = p
-		return nil
-	}
-	return fmt.Errorf("package %q not found", pkg)
-}
-
-func (fgl *fakegolist) cleanup() {
-	os.RemoveAll(fgl.dir)
-	os.Remove(fgl.binfile)
-}
-
-func TestIsTestStale(t *testing.T) {
-	cases := []struct {
-		fileMtime    map[string]time.Time
-		pkgStaleness map[string]bool
-		result       bool
-	}{
-		// Basic test: binary is fresh, all modifications were before the binary was built.
-		{
-			result: false,
-		},
-		// A local test file is new, hence binary must be stale.
-		{
-			fileMtime: map[string]time.Time{
-				"foo_test.go": time.Now().Add(1 * time.Hour),
-			},
-			result: true,
-		},
-		// Test package is new, so binary must be stale.
-		{
-			pkgStaleness: map[string]bool{
-				"example.com/proj/pkg/test": true,
-			},
-			result: true,
-		},
-		// Test package dependencies are new, so binary must be stale.
-		{
-			pkgStaleness: map[string]bool{
-				"example.com/proj/cmd/p3/c12/c23": true,
-				"strings":                         true,
-			},
-			result: true,
-		},
-		// External test files are new, hence binary must be stale.
-		{
-			fileMtime: map[string]time.Time{
-				"xfoo_test.go": time.Now().Add(1 * time.Hour),
-				"xbar_test.go": time.Now().Add(2 * time.Hour),
-			},
-			result: true,
-		},
-		// External test dependency is new, so binary must be stale.
-		{
-			pkgStaleness: map[string]bool{
-				"os": true,
-			},
-			result: true,
-		},
-		// Multiple source files and dependencies are new, so binary must be stale.
-		{
-			fileMtime: map[string]time.Time{
-				"foo_test.go":  time.Now().Add(1 * time.Hour),
-				"xfoo_test.go": time.Now().Add(2 * time.Hour),
-				"xbar_test.go": time.Now().Add(3 * time.Hour),
-			},
-			pkgStaleness: map[string]bool{
-				"example.com/proj/pkg/p1":         true,
-				"example.com/proj/pkg/p1/c11":     true,
-				"example.com/proj/pkg/p2":         true,
-				"example.com/proj/cmd/p3/c12/c23": true,
-				"strings":                         true,
-				"os":                              true,
-			},
-			result: true,
-		},
-		// Everything is new, so binary must be stale.
-		{
-			fileMtime: map[string]time.Time{
-				"foo_test.go":  time.Now().Add(3 * time.Hour),
-				"bar_test.go":  time.Now().Add(1 * time.Hour),
-				"xfoo_test.go": time.Now().Add(2 * time.Hour),
-				"xbar_test.go": time.Now().Add(1 * time.Hour),
-				"xbaz_test.go": time.Now().Add(2 * time.Hour),
-			},
-			pkgStaleness: map[string]bool{
-				"example.com/proj/pkg/p1":         true,
-				"example.com/proj/pkg/p1/c11":     true,
-				"example.com/proj/pkg/p2":         true,
-				"example.com/proj/cmd/p3/c12/c23": true,
-				"example.com/proj/pkg/test":       true,
-				"strings":                         true,
-				"testing":                         true,
-				"os":                              true,
-			},
-			result: true,
-		},
-	}
-
-	for _, tc := range cases {
-		fgl, err := newFakegolist()
-		if err != nil {
-			t.Fatalf("failed to setup the test: %v", err)
-		}
-		defer fgl.cleanup()
-
-		for fn, mtime := range tc.fileMtime {
-			if err := fgl.chMtime(fn, mtime); err != nil {
-				t.Fatalf("failed to change the mtime of %q: %v", fn, err)
-			}
-		}
-
-		for pkg, stale := range tc.pkgStaleness {
-			if err := fgl.chStale(pkg, stale); err != nil {
-				t.Fatalf("failed to change the staleness of %q: %v", pkg, err)
-			}
-		}
-
-		if tc.result != isTestStale(fgl, fgl.binfile, testPkg) {
-			if tc.result {
-				t.Errorf("Expected test package %q to be stale", testPkg)
-			} else {
-				t.Errorf("Expected test package %q to be not stale", testPkg)
-			}
-		}
-	}
-}

hack/generate-bindata.sh

@@ -18,10 +18,8 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-if [[ -z "${KUBE_ROOT:-}" ]]; then
+export KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-	KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
-fi
-
 source "${KUBE_ROOT}/hack/lib/logging.sh"
 
 if [[ ! -d "${KUBE_ROOT}/examples" ]]; then
@@ -31,7 +29,7 @@ fi
 
 # kube::golang::build_kube_toolchain installs the vendored go-bindata in
 # $GOPATH/bin, so make sure that's explicitly part of our $PATH.
-export PATH="${GOPATH}/bin:${PATH}"
+export PATH="${KUBE_OUTPUT_BINPATH}:${PATH}"
 
 if ! which go-bindata &>/dev/null ; then
 	echo "Cannot find go-bindata."

hack/lib/golang.sh

@@ -213,11 +213,6 @@ readonly KUBE_STATIC_LIBRARIES=(
   kubectl
 )
 
-# Add any files with those //generate annotations in the array below.
-readonly KUBE_BINDATAS=(
-  test/e2e/generated/gobindata_util.go
-)
-
 kube::golang::is_statically_linked_library() {
   local e
   for e in "${KUBE_STATIC_LIBRARIES[@]}"; do [[ "$1" == *"/$e" ]] && return 0; done;
@@ -355,6 +350,7 @@ kube::golang::setup_env() {
   kube::golang::create_gopath_tree
 
   export GOPATH="${KUBE_GOPATH}"
+  export GOCACHE="${KUBE_GOPATH}/cache"
 
   # Append KUBE_EXTRA_GOPATH to the GOPATH if it is defined.
   if [[ -n ${KUBE_EXTRA_GOPATH:-} ]]; then
@@ -416,52 +412,9 @@ kube::golang::place_bins() {
   done
 }
 
-kube::golang::fallback_if_stdlib_not_installable() {
-  local go_root_dir=$(go env GOROOT);
-  local go_host_os=$(go env GOHOSTOS);
-  local go_host_arch=$(go env GOHOSTARCH);
-  local cgo_pkg_dir=${go_root_dir}/pkg/${go_host_os}_${go_host_arch}_cgo;
-
-  if [ -e ${cgo_pkg_dir} ]; then
-    return 0;
-  fi
-
-  if [ -w ${go_root_dir}/pkg ]; then
-    return 0;
-  fi
-
-  kube::log::status "+++ Warning: stdlib pkg with cgo flag not found.";
-  kube::log::status "+++ Warning: stdlib pkg cannot be rebuilt since ${go_root_dir}/pkg is not writable by `whoami`";
-  kube::log::status "+++ Warning: Make ${go_root_dir}/pkg writable for `whoami` for a one-time stdlib install, Or"
-  kube::log::status "+++ Warning: Rebuild stdlib using the command 'CGO_ENABLED=0 go install -a -installsuffix cgo std'";
-  kube::log::status "+++ Falling back to go build, which is slower";
-
-  use_go_build=true
-}
-
-# Builds the toolchain necessary for building kube. This needs to be
-# built only on the host platform.
-# TODO: Find this a proper home.
-# Ideally, not a shell script because testing shell scripts is painful.
-kube::golang::build_kube_toolchain() {
-  local targets=(
-    hack/cmd/teststale
-    vendor/github.com/jteeuwen/go-bindata/go-bindata
-  )
-
-  local binaries
-  binaries=($(kube::golang::binaries_from_targets "${targets[@]}"))
-
-  kube::log::status "Building the toolchain targets:" "${binaries[@]}"
-  go install "${goflags[@]:+${goflags[@]}}" \
-        -gcflags "${gogcflags}" \
-        -ldflags "${goldflags}" \
-        "${binaries[@]:+${binaries[@]}}"
-}
-
 # Try and replicate the native binary placement of go install without
 # calling go install.
-kube::golang::output_filename_for_binary() {
+kube::golang::outfile_for_binary() {
   local binary=$1
   local platform=$2
   local output_path="${KUBE_GOPATH}/bin"
@@ -477,7 +430,6 @@ kube::golang::output_filename_for_binary() {
 
 kube::golang::build_binaries_for_platform() {
   local platform=$1
-  local use_go_build=${2-}
 
   local -a statics=()
   local -a nonstatics=()
@@ -496,79 +448,25 @@ kube::golang::build_binaries_for_platform() {
   done
 
   if [[ "${#statics[@]}" != 0 ]]; then
-      kube::golang::fallback_if_stdlib_not_installable;
+    CGO_ENABLED=0 go install -installsuffix static "${goflags[@]:+${goflags[@]}}" \
+      -gcflags "${gogcflags}" \
+      -ldflags "${goldflags}" \
+      "${statics[@]:+${statics[@]}}"
   fi
 
-  if [[ -n ${use_go_build:-} ]]; then
+  if [[ "${#nonstatics[@]}" != 0 ]]; then
-    kube::log::progress "    "
+    go install "${goflags[@]:+${goflags[@]}}" \
-    for binary in "${statics[@]:+${statics[@]}}"; do
+      -gcflags "${gogcflags}" \
-      local outfile=$(kube::golang::output_filename_for_binary "${binary}" "${platform}")
+      -ldflags "${goldflags}" \
-      CGO_ENABLED=0 go build -o "${outfile}" \
+      "${nonstatics[@]:+${nonstatics[@]}}"
-        "${goflags[@]:+${goflags[@]}}" \
-        -gcflags "${gogcflags}" \
-        -ldflags "${goldflags}" \
-        "${binary}"
-      kube::log::progress "*"
-    done
-    for binary in "${nonstatics[@]:+${nonstatics[@]}}"; do
-      local outfile=$(kube::golang::output_filename_for_binary "${binary}" "${platform}")
-      go build -o "${outfile}" \
-        "${goflags[@]:+${goflags[@]}}" \
-        -gcflags "${gogcflags}" \
-        -ldflags "${goldflags}" \
-        "${binary}"
-      kube::log::progress "*"
-    done
-    kube::log::progress "\n"
-  else
-    # Use go install.
-    if [[ "${#nonstatics[@]}" != 0 ]]; then
-      go install "${goflags[@]:+${goflags[@]}}" \
-        -gcflags "${gogcflags}" \
-        -ldflags "${goldflags}" \
-        "${nonstatics[@]:+${nonstatics[@]}}"
-    fi
-    if [[ "${#statics[@]}" != 0 ]]; then
-      CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \
-        -gcflags "${gogcflags}" \
-        -ldflags "${goldflags}" \
-        "${statics[@]:+${statics[@]}}"
-    fi
   fi
 
   for test in "${tests[@]:+${tests[@]}}"; do
-    local outfile=$(kube::golang::output_filename_for_binary "${test}" \
+    local outfile=$(kube::golang::outfile_for_binary "${test}" "${platform}")
-      "${platform}")
-
     local testpkg="$(dirname ${test})"
 
-    # Staleness check always happens on the host machine, so we don't
-    # have to locate the `teststale` binaries for the other platforms.
-    # Since we place the host binaries in `$KUBE_GOPATH/bin`, we can
-    # assume that the binary exists there, if it exists at all.
-    # Otherwise, something has gone wrong with building the `teststale`
-    # binary and we should safely proceed building the test binaries
-    # assuming that they are stale. There is no good reason to error
-    # out.
-    if test -x "${KUBE_GOPATH}/bin/teststale" && ! "${KUBE_GOPATH}/bin/teststale" -binary "${outfile}" -package "${testpkg}"
-    then
-      continue
-    fi
-
-    # `go test -c` below directly builds the binary. It builds the packages,
-    # but it never installs them. `go test -i` only installs the dependencies
-    # of the test, but not the test package itself. So neither `go test -c`
-    # nor `go test -i` installs, for example, test/e2e.a. And without that,
-    # doing a staleness check on k8s.io/kubernetes/test/e2e package always
-    # returns true (always stale). And that's why we need to install the
-    # test package.
-    go install "${goflags[@]:+${goflags[@]}}" \
-        -gcflags "${gogcflags}" \
-        -ldflags "${goldflags}" \
-        "${testpkg}"
-
     mkdir -p "$(dirname ${outfile})"
-    go test -i -c \
+    go test -c \
       "${goflags[@]:+${goflags[@]}}" \
       -gcflags "${gogcflags}" \
       -ldflags "${goldflags}" \
@@ -629,14 +527,11 @@ kube::golang::build_binaries() {
     goldflags="${GOLDFLAGS:-} $(kube::version::ldflags)"
     gogcflags="${GOGCFLAGS:-}"
 
-    local use_go_build
     local -a targets=()
     local arg
 
     for arg; do
-      if [[ "${arg}" == "--use_go_build" ]]; then
+      if [[ "${arg}" == -* ]]; then
-        use_go_build=true
-      elif [[ "${arg}" == -* ]]; then
         # Assume arguments starting with a dash are flags to pass to go.
         goflags+=("${arg}")
       else
@@ -671,26 +566,14 @@ kube::golang::build_binaries() {
       fi
     fi
 
-    # First build the toolchain before building any other targets
-    kube::golang::build_kube_toolchain
-
-    kube::log::status "Generating bindata:" "${KUBE_BINDATAS[@]}"
-    for bindata in "${KUBE_BINDATAS[@]}"; do
-      # Only try to generate bindata if the file exists, since in some cases
-      # one-off builds of individual directories may exclude some files.
-      if [[ -f "${KUBE_ROOT}/${bindata}" ]]; then
-        go generate "${goflags[@]:+${goflags[@]}}" "${KUBE_ROOT}/${bindata}"
-      fi
-    done
-
     if [[ "${parallel}" == "true" ]]; then
       kube::log::status "Building go targets for {${platforms[*]}} in parallel (output will appear in a burst when complete):" "${targets[@]}"
       local platform
       for platform in "${platforms[@]}"; do (
           kube::golang::set_platform_envs "${platform}"
-          kube::log::status "${platform}: go build started"
+          kube::log::status "${platform}: build started"
-          kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
+          kube::golang::build_binaries_for_platform ${platform}
-          kube::log::status "${platform}: go build finished"
+          kube::log::status "${platform}: build finished"
         ) &> "/tmp//${platform//\//_}.build" &
       done
 
@@ -709,7 +592,7 @@ kube::golang::build_binaries() {
         kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
         (
           kube::golang::set_platform_envs "${platform}"
-          kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
+          kube::golang::build_binaries_for_platform ${platform}
         )
       done
     fi

hack/make-rules/test.sh

@@ -278,12 +278,6 @@ runTests() {
   # command, which is much faster.
   if [[ ! ${KUBE_COVER} =~ ^[yY]$ ]]; then
     kube::log::status "Running tests without code coverage"
-    # `go test` does not install the things it builds. `go test -i` installs
-    # the build artifacts but doesn't run the tests.  The two together provide
-    # a large speedup for tests that do not need to be rebuilt.
-    go test -i "${goflags[@]:+${goflags[@]}}" \
-      ${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
-     "${testargs[@]:+${testargs[@]}}"
     go test "${goflags[@]:+${goflags[@]}}" \
       ${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
      "${testargs[@]:+${testargs[@]}}" \
@@ -319,21 +313,11 @@ runTests() {
   for path in $(echo $cover_ignore_dirs | sed 's/|/ /g'); do
       echo -e "skipped\tk8s.io/kubernetes/$path"
   done
-  #
+
-  # `go test` does not install the things it builds. `go test -i` installs
-  # the build artifacts but doesn't run the tests.  The two together provide
-  # a large speedup for tests that do not need to be rebuilt.
   printf "%s\n" "${@}" \
     | grep -Ev $cover_ignore_dirs \
     | xargs -I{} -n 1 -P ${KUBE_COVERPROCS} \
     bash -c "set -o pipefail; _pkg=\"\$0\"; _pkg_out=\${_pkg//\//_}; \
-      go test -i ${goflags[@]:+${goflags[@]}} \
-        ${KUBE_RACE} \
-        ${KUBE_TIMEOUT} \
-        -cover -covermode=\"${KUBE_COVERMODE}\" \
-        -coverprofile=\"${cover_report_dir}/\${_pkg}/${cover_profile}\" \
-        \"\${_pkg}\" \
-        ${testargs[@]:+${testargs[@]}}
       go test ${goflags[@]:+${goflags[@]}} \
         ${KUBE_RACE} \
         ${KUBE_TIMEOUT} \

hack/verify-typecheck.sh

@@ -25,9 +25,7 @@ kube::golang::verify_go_version
 
 cd "${KUBE_ROOT}"
 
-if [[ ! -f test/e2e/generated/bindata.go ]]; then
+make --no-print-directory -C "${KUBE_ROOT}" generated_files
-  make --no-print-directory -C "${KUBE_ROOT}" verify_generated_files
-fi
 
 ret=0
 go run test/typecheck/main.go "$@" || ret=$?

test/e2e/generated/gobindata_util.go

@@ -16,8 +16,6 @@ limitations under the License.
 
 package generated
 
-//go:generate ../../../hack/generate-bindata.sh
-
 import "github.com/golang/glog"
 
 /*

test/test_owners.csv

@@ -544,7 +544,6 @@ k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protobuf,smarterclayton,0,
 k8s.io/kubernetes/cmd/libs/go2idl/openapi-gen/generators,davidopp,1,
 k8s.io/kubernetes/examples,Random-Liu,0,
 k8s.io/kubernetes/hack,thockin,1,
-k8s.io/kubernetes/hack/cmd/teststale,thockin,1,
 k8s.io/kubernetes/pkg/api,Q-Lee,1,
 k8s.io/kubernetes/pkg/api/endpoints,cjcullen,1,
 k8s.io/kubernetes/pkg/api/events,jlowdermilk,1,