Honor per-project prune options correctly

CHANGELOG.md

@@ -4,10 +4,16 @@ NEW FEATURES:
 
 BUG FIXES:
 
-* Fix per-project prune option handling ([#1562](https://github.com/golang/dep/pull/1562))
+* Fix per-project prune option handling ([#1570](https://github.com/golang/dep/pull/1570))
 
 IMPROVEMENTS:
 
+# v0.4.1
+
+BUG FIXES:
+
+* Fix per-project prune option handling ([#1570](https://github.com/golang/dep/pull/1570))
+
 # v0.4.0
 
 NEW FEATURES:

cmd/dep/init.go

@@ -119,7 +119,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
 	}
 
 	// Set default prune options for go-tests and unused-packages
-	p.Manifest.PruneOptions.PruneOptions = gps.PruneNestedVendorDirs + gps.PruneGoTestFiles + gps.PruneUnusedPackages
+	p.Manifest.PruneOptions.DefaultOptions = gps.PruneNestedVendorDirs | gps.PruneGoTestFiles | gps.PruneUnusedPackages
 
 	if cmd.gopath {
 		gs := newGopathScanner(ctx, directDeps, sm)

cmd/dep/prune.go

@@ -5,9 +5,20 @@
 package main
 
 import (
+	"bytes"
 	"flag"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
 
 	"github.com/golang/dep"
+	"github.com/golang/dep/gps"
+	"github.com/golang/dep/gps/pkgtree"
+	"github.com/golang/dep/internal/fs"
+	"github.com/pkg/errors"
 )
 
 const pruneShortHelp = `Pruning is now performed automatically by dep ensure.`
@@ -17,20 +28,182 @@ Set prune options in the manifest and it will be applied after every ensure.
 dep prune will be removed in a future version of dep, causing this command to exit non-0.
 `
 
-type pruneCommand struct{}
+type pruneCommand struct {
+}
 
 func (cmd *pruneCommand) Name() string      { return "prune" }
 func (cmd *pruneCommand) Args() string      { return "" }
 func (cmd *pruneCommand) ShortHelp() string { return pruneShortHelp }
 func (cmd *pruneCommand) LongHelp() string  { return pruneLongHelp }
-func (cmd *pruneCommand) Hidden() bool      { return true }
+func (cmd *pruneCommand) Hidden() bool      { return false }
 
-func (cmd *pruneCommand) Register(fs *flag.FlagSet) {}
+func (cmd *pruneCommand) Register(fs *flag.FlagSet) {
+}
 
 func (cmd *pruneCommand) Run(ctx *dep.Ctx, args []string) error {
 	ctx.Out.Printf("Pruning is now performed automatically by dep ensure.\n")
 	ctx.Out.Printf("Set prune settings in %s and it it will be applied when running ensure.\n", dep.ManifestName)
 	ctx.Out.Printf("\ndep prune will be removed in a future version, and this command will exit non-0.\nPlease update your scripts.\n")
 
+	p, err := ctx.LoadProject()
+	if err != nil {
+		return err
+	}
+
+	sm, err := ctx.SourceManager()
+	if err != nil {
+		return err
+	}
+	sm.UseDefaultSignalHandling()
+	defer sm.Release()
+
+	// While the network churns on ListVersions() requests, statically analyze
+	// code from the current project.
+	ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot))
+	if err != nil {
+		return errors.Wrap(err, "analysis of local packages failed: %v")
+	}
+
+	// Set up a solver in order to check the InputHash.
+	params := p.MakeParams()
+	params.RootPackageTree = ptree
+
+	if ctx.Verbose {
+		params.TraceLogger = ctx.Err
+	}
+
+	s, err := gps.Prepare(params, sm)
+	if err != nil {
+		return errors.Wrap(err, "could not set up solver for input hashing")
+	}
+
+	if p.Lock == nil {
+		return errors.Errorf("Gopkg.lock must exist for prune to know what files are safe to remove.")
+	}
+
+	if !bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
+		return errors.Errorf("Gopkg.lock is out of sync; run dep ensure before pruning.")
+	}
+
+	pruneLogger := ctx.Err
+	if !ctx.Verbose {
+		pruneLogger = log.New(ioutil.Discard, "", 0)
+	}
+	return pruneProject(p, sm, pruneLogger)
+}
+
+// pruneProject removes unused packages from a project.
+func pruneProject(p *dep.Project, sm gps.SourceManager, logger *log.Logger) error {
+	td, err := ioutil.TempDir(os.TempDir(), "dep")
+	if err != nil {
+		return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
+	}
+	defer os.RemoveAll(td)
+
+	if err := gps.WriteDepTree(td, p.Lock, sm, gps.CascadingPruneOptions{DefaultOptions: gps.PruneNestedVendorDirs}, logger); err != nil {
+		return err
+	}
+
+	var toKeep []string
+	for _, project := range p.Lock.Projects() {
+		projectRoot := string(project.Ident().ProjectRoot)
+		for _, pkg := range project.Packages() {
+			toKeep = append(toKeep, filepath.Join(projectRoot, pkg))
+		}
+	}
+
+	toDelete, err := calculatePrune(td, toKeep, logger)
+	if err != nil {
+		return err
+	}
+
+	if len(toDelete) > 0 {
+		logger.Println("Calculated the following directories to prune:")
+		for _, d := range toDelete {
+			logger.Printf("  %s\n", d)
+		}
+	} else {
+		logger.Println("No directories found to prune")
+	}
+
+	if err := deleteDirs(toDelete); err != nil {
+		return err
+	}
+
+	vpath := filepath.Join(p.AbsRoot, "vendor")
+	vendorbak := vpath + ".orig"
+	var failerr error
+	if _, err := os.Stat(vpath); err == nil {
+		// Move out the old vendor dir. just do it into an adjacent dir, to
+		// try to mitigate the possibility of a pointless cross-filesystem
+		// move with a temp directory.
+		if _, err := os.Stat(vendorbak); err == nil {
+			// If the adjacent dir already exists, bite the bullet and move
+			// to a proper tempdir.
+			vendorbak = filepath.Join(td, "vendor.orig")
+		}
+		failerr = fs.RenameWithFallback(vpath, vendorbak)
+		if failerr != nil {
+			goto fail
+		}
+	}
+
+	// Move in the new one.
+	failerr = fs.RenameWithFallback(td, vpath)
+	if failerr != nil {
+		goto fail
+	}
+
+	os.RemoveAll(vendorbak)
+
 	return nil
+
+fail:
+	fs.RenameWithFallback(vendorbak, vpath)
+	return failerr
 }
+
+func calculatePrune(vendorDir string, keep []string, logger *log.Logger) ([]string, error) {
+	logger.Println("Calculating prune. Checking the following packages:")
+	sort.Strings(keep)
+	toDelete := []string{}
+	err := filepath.Walk(vendorDir, func(path string, info os.FileInfo, err error) error {
+		if _, err := os.Lstat(path); err != nil {
+			return nil
+		}
+		if !info.IsDir() {
+			return nil
+		}
+		if path == vendorDir {
+			return nil
+		}
+
+		name := strings.TrimPrefix(path, vendorDir+string(filepath.Separator))
+		logger.Printf("  %s", name)
+		i := sort.Search(len(keep), func(i int) bool {
+			return name <= keep[i]
+		})
+		if i >= len(keep) || !strings.HasPrefix(keep[i], name) {
+			toDelete = append(toDelete, path)
+		}
+		return nil
+	})
+	return toDelete, err
+}
+
+func deleteDirs(toDelete []string) error {
+	// sort by length so we delete sub dirs first
+	sort.Sort(byLen(toDelete))
+	for _, path := range toDelete {
+		if err := os.RemoveAll(path); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type byLen []string
+
+func (a byLen) Len() int           { return len(a) }
+func (a byLen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }

gps/prune.go

@@ -18,16 +18,6 @@ import (
 // PruneOptions represents the pruning options used to write the dependecy tree.
 type PruneOptions uint8
 
-// PruneProjectOptions is map of prune options per project name.
-type PruneProjectOptions map[ProjectRoot]PruneOptions
-
-// RootPruneOptions represents the root prune options for the project.
-// It contains the global options and a map of options per project.
-type RootPruneOptions struct {
-	PruneOptions   PruneOptions
-	ProjectOptions PruneProjectOptions
-}
-
 const (
 	// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
 	PruneNestedVendorDirs PruneOptions = 1 << iota
@@ -41,24 +31,85 @@ const (
 	PruneGoTestFiles
 )
 
-// DefaultRootPruneOptions instantiates a copy of the default root prune options.
-func DefaultRootPruneOptions() RootPruneOptions {
-	return RootPruneOptions{
-		PruneOptions:   PruneNestedVendorDirs,
-		ProjectOptions: PruneProjectOptions{},
-	}
+// PruneOptionSet represents trinary distinctions for each of the types of
+// prune rules (as expressed via PruneOptions): nested vendor directories,
+// unused packages, non-go files, and go test files.
+//
+// The three-way distinction is between "none", "true", and "false", represented
+// by uint8 values of 0, 1, and 2, respectively.
+//
+// This trinary distinction is necessary in order to record, with full fidelity,
+// a cascading tree of pruning values, as expressed in CascadingPruneOptions; a
+// simple boolean cannot delineate between "false" and "none".
+type PruneOptionSet struct {
+	NestedVendor   uint8
+	UnusedPackages uint8
+	NonGoFiles     uint8
+	GoTests        uint8
+}
+
+// CascadingPruneOptions is a set of rules for pruning a dependency tree.
+//
+// The DefaultOptions are the global default pruning rules, expressed as a
+// single PruneOptions bitfield. These global rules will cascade down to
+// individual project rules, unless superseded.
+type CascadingPruneOptions struct {
+	DefaultOptions    PruneOptions
+	PerProjectOptions map[ProjectRoot]PruneOptionSet
 }
 
-// PruneOptionsFor returns the prune options for the passed project root.
+// PruneOptionsFor returns the PruneOptions bits for the given project,
+// indicating which pruning rules should be applied to the project's code.
 //
-// It will return the root prune options if the project does not have specific
-// options or if it does not exist in the manifest.
-func (o *RootPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
-	if po, ok := o.ProjectOptions[pr]; ok {
-		return po
+// It computes the cascade from default to project-specific options (if any) on
+// the fly.
+func (o CascadingPruneOptions) PruneOptionsFor(pr ProjectRoot) PruneOptions {
+	po, has := o.PerProjectOptions[pr]
+	if !has {
+		return o.DefaultOptions
+	}
+
+	ops := o.DefaultOptions
+	if po.NestedVendor != 0 {
+		if po.NestedVendor == 1 {
+			ops |= PruneNestedVendorDirs
+		} else {
+			ops &^= PruneNestedVendorDirs
+		}
 	}
 
-	return o.PruneOptions
+	if po.UnusedPackages != 0 {
+		if po.UnusedPackages == 1 {
+			ops |= PruneUnusedPackages
+		} else {
+			ops &^= PruneUnusedPackages
+		}
+	}
+
+	if po.NonGoFiles != 0 {
+		if po.NonGoFiles == 1 {
+			ops |= PruneNonGoFiles
+		} else {
+			ops &^= PruneNonGoFiles
+		}
+	}
+
+	if po.GoTests != 0 {
+		if po.GoTests == 1 {
+			ops |= PruneGoTestFiles
+		} else {
+			ops &^= PruneGoTestFiles
+		}
+	}
+
+	return ops
+}
+
+func defaultCascadingPruneOptions() CascadingPruneOptions {
+	return CascadingPruneOptions{
+		DefaultOptions:    PruneNestedVendorDirs,
+		PerProjectOptions: map[ProjectRoot]PruneOptionSet{},
+	}
 }
 
 var (