Merge pull request #1271 from sdboyer/pkgtree-hidden-trimming

Trim hidden packages from (root) PackageTrees

cmd/dep/ensure.go

@@ -178,12 +178,12 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error {
 		return cmd.runVendorOnly(ctx, args, p, sm, params)
 	}
 
-	params.RootPackageTree, err = pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	params.RootPackageTree, err = p.ParseRootPackageTree()
 	if err != nil {
-		return errors.Wrap(err, "ensure ListPackage for project")
+		return err
 	}
 
-	if fatal, err := checkErrors(params.RootPackageTree.Packages); err != nil {
+	if fatal, err := checkErrors(params.RootPackageTree.Packages, p.Manifest.IgnoredPackages()); err != nil {
 		if fatal {
 			return err
 		} else if ctx.Verbose {
@@ -761,13 +761,17 @@ func getProjectConstraint(arg string, sm gps.SourceManager) (gps.ProjectConstrai
 	return gps.ProjectConstraint{Ident: pi, Constraint: c}, arg, nil
 }
 
-func checkErrors(m map[string]pkgtree.PackageOrErr) (fatal bool, err error) {
+func checkErrors(m map[string]pkgtree.PackageOrErr, ignore *pkgtree.IgnoredRuleset) (fatal bool, err error) {
 	var (
 		noGoErrors    int
 		pkgtreeErrors = make(pkgtreeErrs, 0, len(m))
 	)
 
-	for _, poe := range m {
+	for ip, poe := range m {
+		if ignore.IsIgnored(ip) {
+			continue
+		}
+
 		if poe.Err != nil {
 			switch poe.Err.(type) {
 			case *build.NoGoError:

cmd/dep/ensure_test.go

@@ -135,7 +135,7 @@ func TestCheckErrors(t *testing.T) {
 
 	for _, tc := range tt {
 		t.Run(tc.name, func(t *testing.T) {
-			fatal, err := checkErrors(tc.pkgOrErrMap)
+			fatal, err := checkErrors(tc.pkgOrErrMap, nil)
 			if tc.fatal != fatal {
 				t.Fatalf("expected fatal flag to be %T, got %T", tc.fatal, fatal)
 			}

cmd/dep/init.go

@@ -227,9 +227,9 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 }
 
 func getDirectDependencies(sm gps.SourceManager, p *dep.Project) (pkgtree.PackageTree, map[string]bool, error) {
-	pkgT, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	pkgT, err := p.ParseRootPackageTree()
 	if err != nil {
-		return pkgtree.PackageTree{}, nil, errors.Wrap(err, "gps.ListPackages")
+		return pkgtree.PackageTree{}, nil, err
 	}
 
 	directDeps := map[string]bool{}

cmd/dep/status.go

@@ -171,7 +171,8 @@ type dotOutput struct {
 func (out *dotOutput) BasicHeader() {
 	out.g = new(graphviz).New()
 
-	ptree, _ := pkgtree.ListPackages(out.p.ResolvedAbsRoot, string(out.p.ImportRoot))
+	ptree, _ := out.p.ParseRootPackageTree()
+	// TODO(sdboyer) should be true, true, false, out.p.Manifest.IgnoredPackages()
 	prm, _ := ptree.ToReachMap(true, false, false, nil)
 
 	out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath))
@@ -358,9 +359,9 @@ type MissingStatus struct {
 func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (hasMissingPkgs bool, errCount int, err error) {
 	// While the network churns on ListVersions() requests, statically analyze
 	// code from the current project.
-	ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	ptree, err := p.ParseRootPackageTree()
 	if err != nil {
-		return false, 0, errors.Wrapf(err, "analysis of local packages failed")
+		return false, 0, err
 	}
 
 	// Set up a solver in order to check the InputHash.
@@ -437,7 +438,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
 						errListPkgCh <- err
 					}
 
-					prm, _ := ptr.ToReachMap(true, false, false, nil)
+					prm, _ := ptr.ToReachMap(true, true, false, p.Manifest.IgnoredPackages())
 					bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
 				}
 

internal/gps/_testdata/src/varied_hidden/.onlyfromtests/a.go

@@ -0,0 +1,18 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package onlyfromtests
+
+import (
+	"sort"
+
+	_ "varied/_secondorder"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	M = sort.Strings
+	_ = gps.Solve
+)

internal/gps/_testdata/src/varied_hidden/.onlyfromtests/b.go

@@ -0,0 +1,15 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package onlyfromtests
+
+import (
+	"os"
+	"sort"
+)
+
+var (
+	_ = sort.Strings
+	_ = os.PathSeparator
+)

internal/gps/_testdata/src/varied_hidden/_frommain/a.go

@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package frommain
+
+import (
+	"sort"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	M = sort.Strings
+	_ = gps.Solve
+)

internal/gps/_testdata/src/varied_hidden/_never/a.go

@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package never
+
+import (
+	"sort"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	M = sort.Strings
+	_ = gps.Solve
+)

internal/gps/_testdata/src/varied_hidden/_secondorder/secondorder.go

@@ -0,0 +1,11 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package secondorder
+
+import "hash"
+
+var (
+	H = hash.Hash
+)

internal/gps/_testdata/src/varied_hidden/always/always_test.go

@@ -0,0 +1,7 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package always
+
+import _ "varied/.onlyfromtests"

internal/gps/_testdata/src/varied_hidden/dotdotslash/a.go

@@ -0,0 +1,13 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dotslash
+
+import (
+	"../github.com/golang/dep/internal/gps"
+)
+
+var (
+	A = gps.Solver
+)

internal/gps/_testdata/src/varied_hidden/locals.go

@@ -0,0 +1,14 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "varied/_frommain"
+	"varied/simple"
+)
+
+var (
+	_ = simple.S
+)

internal/gps/_testdata/src/varied_hidden/main.go

@@ -0,0 +1,13 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"net/http"
+)
+
+var (
+	_ = http.Client
+)

internal/gps/_testdata/src/varied_hidden/simple/locals.go

@@ -0,0 +1,11 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simple
+
+import "varied/simple/testdata"
+
+var (
+	_ = testdata.H
+)

internal/gps/_testdata/src/varied_hidden/simple/simple.go

@@ -0,0 +1,16 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package simple
+
+import (
+	"go/parser"
+
+	"github.com/golang/dep/internal/gps"
+)
+
+var (
+	_ = parser.ParseFile
+	S = gps.Prepare
+)

internal/gps/_testdata/src/varied_hidden/simple/testdata/another.go

@@ -0,0 +1,7 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testdata
+
+import _ "varied/dotdotslash"

internal/gps/pkgtree/ignored_ruleset.go

@@ -0,0 +1,111 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pkgtree
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/armon/go-radix"
+)
+
+// IgnoredRuleset comprises a set of rules for ignoring import paths. It can
+// manage both literal and prefix-wildcard matches.
+type IgnoredRuleset struct {
+	t *radix.Tree
+}
+
+// NewIgnoredRuleset processes a set of strings into an IgnoredRuleset. Strings
+// that end in "*" are treated as wildcards, where any import path with a
+// matching prefix will be ignored. IgnoredRulesets are immutable once created.
+//
+// Duplicate and redundant (i.e. a literal path that has a prefix of a wildcard
+// path) declarations are discarded. Consequently, it is possible that the
+// returned IgnoredRuleset may have a smaller Len() than the input slice.
+func NewIgnoredRuleset(ig []string) *IgnoredRuleset {
+	if len(ig) == 0 {
+		return &IgnoredRuleset{}
+	}
+
+	ir := &IgnoredRuleset{
+		t: radix.New(),
+	}
+
+	// Sort the list of all the ignores in order to ensure that wildcard
+	// precedence is recorded correctly in the trie.
+	sort.Strings(ig)
+	for _, i := range ig {
+		// Skip global ignore and empty string.
+		if i == "*" || i == "" {
+			continue
+		}
+
+		_, wildi, has := ir.t.LongestPrefix(i)
+		// We may not always have a value here, but if we do, then it's a bool.
+		wild, _ := wildi.(bool)
+		// Check if it's a wildcard ignore.
+		if strings.HasSuffix(i, "*") {
+			// Check if it is ineffectual.
+			if has && wild {
+				// Skip ineffectual wildcard ignore.
+				continue
+			}
+			// Create the ignore prefix and insert in the radix tree.
+			ir.t.Insert(i[:len(i)-1], true)
+		} else if !has || !wild {
+			ir.t.Insert(i, false)
+		}
+	}
+
+	if ir.t.Len() == 0 {
+		ir.t = nil
+	}
+
+	return ir
+}
+
+// IsIgnored indicates whether the provided path should be ignored, according to
+// the ruleset.
+func (ir *IgnoredRuleset) IsIgnored(path string) bool {
+	if path == "" || ir == nil || ir.t == nil {
+		return false
+	}
+
+	prefix, wildi, has := ir.t.LongestPrefix(path)
+	return has && (wildi.(bool) || path == prefix)
+}
+
+// Len indicates the number of rules in the ruleset.
+func (ir *IgnoredRuleset) Len() int {
+	if ir == nil || ir.t == nil {
+		return 0
+	}
+
+	return ir.t.Len()
+}
+
+// ToSlice converts the contents of the IgnoredRuleset to a string slice.
+//
+// This operation is symmetrically dual to NewIgnoredRuleset.
+func (ir *IgnoredRuleset) ToSlice() []string {
+	irlen := ir.Len()
+	if irlen == 0 {
+		return nil
+	}
+
+	items := make([]string, 0, irlen)
+	ir.t.Walk(func(s string, v interface{}) bool {
+		if s != "" {
+			if v.(bool) {
+				items = append(items, s+"*")
+			} else {
+				items = append(items, s)
+			}
+		}
+		return false
+	})
+
+	return items
+}