gps: source cache: enable opt-in persistent caching via DEPCACHEAGE env var

CHANGELOG.md

@@ -4,7 +4,8 @@ NEW FEATURES:
 
 * Add CI tests against go1.10. Drop support for go1.8. ([#1620](https://github.com/golang/dep/pull/1620))
 * Added `install.sh` script. ([#1533](https://github.com/golang/dep/pull/1533))
-* List out of date projects in dep status ([#1553](https://github.com/golang/dep/pull/1553)).
+* List out of date projects in dep status. ([#1553](https://github.com/golang/dep/pull/1553)).
+* Enabled opt-in persistent caching via $DEPCACHEAGE env var. ([#1711](https://github.com/golang/dep/pull/1711))
 
 BUG FIXES:
 

cmd/dep/main.go

@@ -18,6 +18,7 @@ import (
 	"runtime/pprof"
 	"strings"
 	"text/tabwriter"
+	"time"
 
 	"github.com/golang/dep"
 	"github.com/golang/dep/internal/fs"
@@ -216,13 +217,24 @@ func (c *Config) Run() int {
 				}
 			}
 
+			var cacheAge time.Duration
+			if env := getEnv(c.Env, "DEPCACHEAGE"); env != "" {
+				var err error
+				cacheAge, err = time.ParseDuration(env)
+				if err != nil {
+					errLogger.Printf("dep: failed to parse $DEPCACHEAGE duration %q: %v\n", env, err)
+					return errorExitCode
+				}
+			}
+
 			// Set up dep context.
 			ctx := &dep.Ctx{
 				Out:            outLogger,
 				Err:            errLogger,
 				Verbose:        *verbose,
 				DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "",
 				Cachedir:       cachedir,
+				CacheAge:       cacheAge,
 			}
 
 			GOPATHS := filepath.SplitList(getEnv(c.Env, "GOPATH"))

context.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"time"
 
 	"github.com/golang/dep/gps"
 	"github.com/golang/dep/internal/fs"
@@ -34,13 +35,14 @@ import (
 //	}
 //
 type Ctx struct {
-	WorkingDir     string      // Where to execute.
-	GOPATH         string      // Selected Go path, containing WorkingDir.
-	GOPATHs        []string    // Other Go paths.
-	Out, Err       *log.Logger // Required loggers.
-	Verbose        bool        // Enables more verbose logging.
-	DisableLocking bool        // When set, no lock file will be created to protect against simultaneous dep processes.
-	Cachedir       string      // Cache directory loaded from environment.
+	WorkingDir     string        // Where to execute.
+	GOPATH         string        // Selected Go path, containing WorkingDir.
+	GOPATHs        []string      // Other Go paths.
+	Out, Err       *log.Logger   // Required loggers.
+	Verbose        bool          // Enables more verbose logging.
+	DisableLocking bool          // When set, no lock file will be created to protect against simultaneous dep processes.
+	Cachedir       string        // Cache directory loaded from environment.
+	CacheAge       time.Duration // Maximum valid age of cached source data. <=0: Don't cache.
 }
 
 // SetPaths sets the WorkingDir and GOPATHs fields. If GOPATHs is empty, then
@@ -99,6 +101,7 @@ func (c *Ctx) SourceManager() (*gps.SourceMgr, error) {
 	}
 
 	return gps.NewSourceManager(gps.SourceManagerConfig{
+		CacheAge:       c.CacheAge,
 		Cachedir:       cachedir,
 		Logger:         c.Out,
 		DisableLocking: c.DisableLocking,

gps/source.go

@@ -67,22 +67,33 @@ type sourceCoordinator struct {
 	psrcmut    sync.Mutex // guards protoSrcs map
 	protoSrcs  map[string][]chan srcReturn
 	cachedir   string
+	cache      sourceCache
 	logger     *log.Logger
 }
 
-func newSourceCoordinator(superv *supervisor, deducer deducer, cachedir string, logger *log.Logger) *sourceCoordinator {
+// newSourceCoordinator returns a new sourceCoordinator.
+// Passing a nil sourceCache defaults to an in-memory cache.
+func newSourceCoordinator(superv *supervisor, deducer deducer, cachedir string, cache sourceCache, logger *log.Logger) *sourceCoordinator {
+	if cache == nil {
+		cache = memoryCache{}
+	}
 	return &sourceCoordinator{
 		supervisor: superv,
 		deducer:    deducer,
 		cachedir:   cachedir,
+		cache:      cache,
 		logger:     logger,
 		srcs:       make(map[string]*sourceGateway),
 		nameToURL:  make(map[string]string),
 		protoSrcs:  make(map[string][]chan srcReturn),
 	}
 }
 
-func (sc *sourceCoordinator) close() {}
+func (sc *sourceCoordinator) close() {
+	if err := sc.cache.close(); err != nil {
+		sc.logger.Println(errors.Wrap(err, "failed to close the source cache"))
+	}
+}
 
 func (sc *sourceCoordinator) getSourceGatewayFor(ctx context.Context, id ProjectIdentifier) (*sourceGateway, error) {
 	if err := sc.supervisor.ctx.Err(); err != nil {
@@ -216,7 +227,8 @@ func (sc *sourceCoordinator) getSourceGatewayFor(ctx context.Context, id Project
 		}
 		src, err := m.try(ctx, sc.cachedir)
 		if err == nil {
-			srcGate, err = newSourceGateway(ctx, src, sc.supervisor, sc.cachedir)
+			cache := sc.cache.newSingleSourceCache(id)
+			srcGate, err = newSourceGateway(ctx, src, sc.supervisor, sc.cachedir, cache)
 			if err == nil {
 				sc.srcs[url] = srcGate
 				break
@@ -260,7 +272,7 @@ type sourceGateway struct {
 
 // newSourceGateway returns a new gateway for src. If the source exists locally,
 // the local state may be cleaned, otherwise we ping upstream.
-func newSourceGateway(ctx context.Context, src source, superv *supervisor, cachedir string) (*sourceGateway, error) {
+func newSourceGateway(ctx context.Context, src source, superv *supervisor, cachedir string, cache singleSourceCache) (*sourceGateway, error) {
 	var state sourceState
 	local := src.existsLocally(ctx)
 	if local {
@@ -276,9 +288,9 @@ func newSourceGateway(ctx context.Context, src source, superv *supervisor, cache
 		srcState: state,
 		src:      src,
 		cachedir: cachedir,
+		cache:    cache,
 		suprvsr:  superv,
 	}
-	sg.cache = sg.createSingleSourceCache()
 
 	if !local {
 		if err := sg.require(ctx, sourceExistsUpstream); err != nil {
@@ -542,14 +554,6 @@ func (sg *sourceGateway) disambiguateRevision(ctx context.Context, r Revision) (
 	return sg.src.disambiguateRevision(ctx, r)
 }
 
-// createSingleSourceCache creates a singleSourceCache instance for use by
-// the encapsulated source.
-func (sg *sourceGateway) createSingleSourceCache() singleSourceCache {
-	// TODO(sdboyer) when persistent caching is ready, just drop in the creation
-	// of a source-specific handle here
-	return newMemoryCache()
-}
-
 // sourceExistsUpstream verifies that the source exists upstream and that the
 // upstreamURL has not changed and returns any additional sourceState, or an error.
 func (sg *sourceGateway) sourceExistsUpstream(ctx context.Context) (sourceState, error) {

gps/source_cache.go

@@ -13,6 +13,16 @@ import (
 	"github.com/golang/dep/gps/pkgtree"
 )
 
+// sourceCache is an interface for creating singleSourceCaches, and safely
+// releasing backing resources via close.
+type sourceCache interface {
+	// newSingleSourceCache creates a new singleSourceCache for id, which
+	// remains valid until close is called.
+	newSingleSourceCache(id ProjectIdentifier) singleSourceCache
+	// close releases background resources.
+	close() error
+}
+
 // singleSourceCache provides a method set for storing and retrieving data about
 // a single source.
 type singleSourceCache interface {
@@ -62,6 +72,15 @@ type singleSourceCache interface {
 	toUnpaired(v Version) (UnpairedVersion, bool)
 }
 
+// memoryCache is a sourceCache which creates singleSourceCacheMemory instances.
+type memoryCache struct{}
+
+func (memoryCache) newSingleSourceCache(ProjectIdentifier) singleSourceCache {
+	return newMemoryCache()
+}
+
+func (memoryCache) close() error { return nil }
+
 type singleSourceCacheMemory struct {
 	// Protects all fields.
 	mut   sync.RWMutex

gps/source_cache_bolt.go

@@ -21,6 +21,10 @@ import (
 	"github.com/pkg/errors"
 )
 
+// boltCacheFilename is a versioned filename for the bolt cache. The version
+// must be incremented whenever incompatible changes are made.
+const boltCacheFilename = "bolt-v1.db"
+
 // boltCache manages a bolt.DB cache and provides singleSourceCaches.
 type boltCache struct {
 	db     *bolt.DB
@@ -30,7 +34,7 @@ type boltCache struct {
 
 // newBoltCache returns a new boltCache backed by a BoltDB file under the cache directory.
 func newBoltCache(cd string, epoch int64, logger *log.Logger) (*boltCache, error) {
-	path := sourceCachePath(cd, "bolt") + ".db"
+	path := filepath.Join(cd, boltCacheFilename)
 	dir := filepath.Dir(path)
 	if fi, err := os.Stat(dir); os.IsNotExist(err) {
 		if err := os.MkdirAll(dir, os.ModeDir|os.ModePerm); err != nil {

gps/source_cache_multi.go

@@ -8,22 +8,72 @@ import (
 	"github.com/golang/dep/gps/pkgtree"
 )
 
-// A multiCache manages two cache levels, ephemeral in-memory and persistent on-disk.
+// multiCache creates singleSourceMultiCaches, and coordinates their async updates.
+type multiCache struct {
+	mem, disk sourceCache
+	// Asynchronous disk cache updates. Closed by the close method.
+	async chan func()
+	// Closed when async has completed processing.
+	done chan struct{}
+}
+
+// newMultiCache returns a new multiCache backed by mem and disk sourceCaches.
+// Spawns a single background goroutine which lives until close() is called.
+func newMultiCache(mem, disk sourceCache) *multiCache {
+	m := &multiCache{
+		mem:   mem,
+		disk:  disk,
+		async: make(chan func(), 50),
+		done:  make(chan struct{}),
+	}
+	go m.processAsync()
+	return m
+}
+
+func (c *multiCache) processAsync() {
+	for f := range c.async {
+		f()
+	}
+	close(c.done)
+}
+
+// close releases resources after blocking until async writes complete.
+func (c *multiCache) close() error {
+	close(c.async)
+	_ = c.mem.close()
+	<-c.done
+	return c.disk.close()
+}
+
+// newSingleSourceCache returns a singleSourceMultiCache for id.
+func (c *multiCache) newSingleSourceCache(id ProjectIdentifier) singleSourceCache {
+	return &singleSourceMultiCache{
+		mem:   c.mem.newSingleSourceCache(id),
+		disk:  c.disk.newSingleSourceCache(id),
+		async: c.async,
+	}
+}
+
+// singleSourceMultiCache manages two cache levels, ephemeral in-memory and persistent on-disk.
 //
 // The in-memory cache is always checked first, with the on-disk used as a fallback.
 // Values read from disk are set in-memory when an appropriate method exists.
 //
-// Set values are cached both in-memory and on-disk.
-type multiCache struct {
+// Set values are cached both in-memory and on-disk. Values are set synchronously
+// in-memory. Writes to the on-disk cache are asynchronous, and executed in order by a
+// background goroutine.
+type singleSourceMultiCache struct {
 	mem, disk singleSourceCache
+	// Asynchronous disk cache updates.
+	async chan<- func()
 }
 
-func (c *multiCache) setManifestAndLock(r Revision, ai ProjectAnalyzerInfo, m Manifest, l Lock) {
+func (c *singleSourceMultiCache) setManifestAndLock(r Revision, ai ProjectAnalyzerInfo, m Manifest, l Lock) {
 	c.mem.setManifestAndLock(r, ai, m, l)
-	c.disk.setManifestAndLock(r, ai, m, l)
+	c.async <- func() { c.disk.setManifestAndLock(r, ai, m, l) }
 }
 
-func (c *multiCache) getManifestAndLock(r Revision, ai ProjectAnalyzerInfo) (Manifest, Lock, bool) {
+func (c *singleSourceMultiCache) getManifestAndLock(r Revision, ai ProjectAnalyzerInfo) (Manifest, Lock, bool) {
 	m, l, ok := c.mem.getManifestAndLock(r, ai)
 	if ok {
 		return m, l, true
@@ -38,12 +88,12 @@ func (c *multiCache) getManifestAndLock(r Revision, ai ProjectAnalyzerInfo) (Man
 	return nil, nil, false
 }
 
-func (c *multiCache) setPackageTree(r Revision, ptree pkgtree.PackageTree) {
+func (c *singleSourceMultiCache) setPackageTree(r Revision, ptree pkgtree.PackageTree) {
 	c.mem.setPackageTree(r, ptree)
-	c.disk.setPackageTree(r, ptree)
+	c.async <- func() { c.disk.setPackageTree(r, ptree) }
 }
 
-func (c *multiCache) getPackageTree(r Revision, pr ProjectRoot) (pkgtree.PackageTree, bool) {
+func (c *singleSourceMultiCache) getPackageTree(r Revision, pr ProjectRoot) (pkgtree.PackageTree, bool) {
 	ptree, ok := c.mem.getPackageTree(r, pr)
 	if ok {
 		return ptree, true
@@ -58,17 +108,17 @@ func (c *multiCache) getPackageTree(r Revision, pr ProjectRoot) (pkgtree.Package
 	return pkgtree.PackageTree{}, false
 }
 
-func (c *multiCache) markRevisionExists(r Revision) {
+func (c *singleSourceMultiCache) markRevisionExists(r Revision) {
 	c.mem.markRevisionExists(r)
-	c.disk.markRevisionExists(r)
+	c.async <- func() { c.disk.markRevisionExists(r) }
 }
 
-func (c *multiCache) setVersionMap(pvs []PairedVersion) {
+func (c *singleSourceMultiCache) setVersionMap(pvs []PairedVersion) {
 	c.mem.setVersionMap(pvs)
-	c.disk.setVersionMap(pvs)
+	c.async <- func() { c.disk.setVersionMap(pvs) }
 }
 
-func (c *multiCache) getVersionsFor(rev Revision) ([]UnpairedVersion, bool) {
+func (c *singleSourceMultiCache) getVersionsFor(rev Revision) ([]UnpairedVersion, bool) {
 	uvs, ok := c.mem.getVersionsFor(rev)
 	if ok {
 		return uvs, true
@@ -77,7 +127,7 @@ func (c *multiCache) getVersionsFor(rev Revision) ([]UnpairedVersion, bool) {
 	return c.disk.getVersionsFor(rev)
 }
 
-func (c *multiCache) getAllVersions() ([]PairedVersion, bool) {
+func (c *singleSourceMultiCache) getAllVersions() ([]PairedVersion, bool) {
 	pvs, ok := c.mem.getAllVersions()
 	if ok {
 		return pvs, true
@@ -92,7 +142,7 @@ func (c *multiCache) getAllVersions() ([]PairedVersion, bool) {
 	return nil, false
 }
 
-func (c *multiCache) getRevisionFor(uv UnpairedVersion) (Revision, bool) {
+func (c *singleSourceMultiCache) getRevisionFor(uv UnpairedVersion) (Revision, bool) {
 	rev, ok := c.mem.getRevisionFor(uv)
 	if ok {
 		return rev, true
@@ -101,7 +151,7 @@ func (c *multiCache) getRevisionFor(uv UnpairedVersion) (Revision, bool) {
 	return c.disk.getRevisionFor(uv)
 }
 
-func (c *multiCache) toRevision(v Version) (Revision, bool) {
+func (c *singleSourceMultiCache) toRevision(v Version) (Revision, bool) {
 	rev, ok := c.mem.toRevision(v)
 	if ok {
 		return rev, true
@@ -110,7 +160,7 @@ func (c *multiCache) toRevision(v Version) (Revision, bool) {
 	return c.disk.toRevision(v)
 }
 
-func (c *multiCache) toUnpaired(v Version) (UnpairedVersion, bool) {
+func (c *singleSourceMultiCache) toUnpaired(v Version) (UnpairedVersion, bool) {
 	uv, ok := c.mem.toUnpaired(v)
 	if ok {
 		return uv, true