@@ -8,10 +8,12 @@ import (
88 "bytes"
99 "cmp"
1010 "context"
11+ "crypto/sha256"
1112 "encoding/json"
1213 "errors"
1314 "fmt"
1415 "io"
16+ "iter"
1517 "log"
1618 "math"
1719 "net"
@@ -37,6 +39,8 @@ import (
3739 "golang.org/x/mod/module"
3840 "golang.org/x/mod/sumdb/dirhash"
3941 "golang.org/x/sync/singleflight"
42+ "tailscale.com/types/result"
43+ "tailscale.com/util/testenv"
4044)
4145
4246const (
@@ -57,10 +61,23 @@ type FS struct {
5761 // It should not have a trailing slash.
5862 ModuleProxyURL string
5963
64+ Logf func (format string , args ... any ) // if non-nil, alternate logger to use
65+
6066 Verbose bool
6167
62- mu sync.RWMutex
63- zipRootCache map [store.ModuleVersion ]modHandleCacheEntry
68+ mu sync.RWMutex
69+ zipRootCache map [store.ModuleVersion ]modHandleCacheEntry
70+ modVerHash map [modVerHash ]store.ModuleVersion // nil until first used
71+ pathHashTarget map [pathHash ]handleTarget // nil until first used
72+ }
73+
74+ func hashModVersion (mv store.ModuleVersion ) (ret modVerHash ) {
75+ s := sha256 .New ()
76+ io .WriteString (s , mv .Module )
77+ io .WriteString (s , "\x00 " )
78+ io .WriteString (s , mv .Version )
79+ s .Sum (ret [:0 ])
80+ return
6481}
6582
6683type modHandleCacheEntry struct {
@@ -147,6 +164,21 @@ func (fs *FS) downloadInfoFile(ctx context.Context, mv store.ModuleVersion) (_ [
147164 return vi .([]byte ), nil
148165}
149166
167+ func (fs * FS ) netLogf (format string , arg ... any ) {
168+ if testenv .InTest () && fs .Client != nil {
169+ format = "[fake-network] " + format
170+ }
171+ fs .logf (format , arg ... )
172+ }
173+
174+ func (fs * FS ) logf (format string , arg ... any ) {
175+ if fs .Logf != nil {
176+ fs .Logf (format , arg ... )
177+ } else {
178+ log .Printf (format , arg ... )
179+ }
180+ }
181+
150182func (fs * FS ) downloadZip (ctx context.Context , mv store.ModuleVersion ) (store.ModHandle , error ) {
151183 baseURL , err := fs .modURLBase (mv )
152184 if err != nil {
@@ -163,7 +195,7 @@ func (fs *FS) downloadZip(ctx context.Context, mv store.ModuleVersion) (store.Mo
163195 return nil , fmt .Errorf ("failed to download %q: %w" , urlStr , err )
164196 }
165197 download [ext ] = data
166- log . Printf ("Downloaded %d bytes from %v" , len (data ), urlStr )
198+ fs . netLogf ("Downloaded %d bytes from %v" , len (data ), urlStr )
167199 }
168200
169201 zr , err := zip .NewReader (bytes .NewReader (download ["zip" ]), int64 (len (download ["zip" ])))
@@ -258,6 +290,12 @@ func (fs *FS) setZipRootCache(mv store.ModuleVersion, h store.ModHandle) {
258290 if fs .zipRootCache == nil {
259291 fs .zipRootCache = make (map [store.ModuleVersion ]modHandleCacheEntry )
260292 }
293+ if fs .modVerHash != nil {
294+ // If it's already initialized, add new stuff to it.
295+ // Otherwise if it's nil, it'll get populated later.
296+ fs .modVerHash [hashModVersion (mv )] = mv
297+ fs .addModuleVersionPathsLocked (mv )
298+ }
261299 now := new (atomic.Int64 )
262300 now .Store (time .Now ().Unix ())
263301 fs .zipRootCache [mv ] = modHandleCacheEntry {
@@ -374,6 +412,158 @@ func (fs *FS) getZiphash(ctx context.Context, mv store.ModuleVersion) (data []by
374412 return fs .Store .GetZipHash (ctx , zr )
375413}
376414
415+ func (fs * FS ) walkStoreModulePaths (ctx context.Context , mh store.ModHandle ) iter.Seq [result.Of [string ]] {
416+ return func (yield func (result.Of [string ]) bool ) {
417+ var doDir func (string ) bool // recursive dir walker, called with "" (root) or "path/to/file-or-dir"
418+ doDir = func (path string ) bool {
419+ if ! yield (result .Value (path )) {
420+ return false
421+ }
422+ ents , err := fs .Store .Readdir (ctx , mh , path )
423+ if err != nil {
424+ yield (result.Error [string ](err ))
425+ return false
426+ }
427+ for _ , ent := range ents {
428+ sub := ent .Name
429+ if path != "" {
430+ sub = path + "/" + ent .Name
431+ }
432+ if ent .Mode .IsDir () {
433+ if ! doDir (sub ) {
434+ return false
435+ }
436+ } else {
437+ if ! yield (result .Value (sub )) {
438+ return false
439+ }
440+ }
441+ }
442+ return true
443+ }
444+ doDir ("" )
445+ }
446+ }
447+
448+ // modVerHash is SHA256(store.ModuleVersion).
449+ type modVerHash [sha256 .Size ]byte
450+
451+ func (h modVerHash ) IsZero () bool { return h == modVerHash {} }
452+
453+ // pathHash is SHA256(either abs path or path-with-a-zp)
454+ type pathHash [sha256 .Size ]byte
455+
456+ var (
457+ cdFileInfo = mkWellKnownPathHash ("info" )
458+ cdFileMod = mkWellKnownPathHash ("mod" )
459+ cdFileZiphash = mkWellKnownPathHash ("ziphash" )
460+ )
461+
462+ func mkWellKnownPathHash (s string ) pathHash {
463+ var h pathHash
464+ copy (h [:], s )
465+ return h
466+ }
467+
468+ // returns (v, nil) on hit, (zero, nil) on miss, or (zero, err) on error.
469+ func (fs * FS ) moduleVersionWithHash (mvh modVerHash ) (mv store.ModuleVersion , err error ) {
470+ var zero store.ModuleVersion
471+
472+ fs .mu .Lock ()
473+ defer fs .mu .Unlock ()
474+ if err := fs .initHandleMapsLocked (); err != nil {
475+ fs .logf ("initHandleMapsLocked error: %v" , err )
476+ return zero , err
477+ }
478+ return fs .modVerHash [mvh ], nil
479+ }
480+
481+ func (fs * FS ) initHandleMapsLocked () error {
482+ if fs .modVerHash != nil {
483+ return nil
484+ }
485+
486+ mvs , err := fs .Store .CachedModules (context .TODO ())
487+ if err != nil {
488+ return fmt .Errorf ("CachedModules: %w" , err )
489+ }
490+
491+ fs .modVerHash = make (map [modVerHash ]store.ModuleVersion )
492+ fs .pathHashTarget = make (map [pathHash ]handleTarget )
493+
494+ for _ , mv := range mvs {
495+ fs .modVerHash [hashModVersion (mv )] = mv
496+ fs .addModuleVersionPathsLocked (mv )
497+ }
498+
499+ for _ , path := range []string {
500+ "cache" ,
501+ "cache/download" ,
502+ statusFile ,
503+ } {
504+ fs .addPathHashTargetLocked (path )
505+ }
506+ for _ , goos := range tsGoGeese {
507+ for _ , goarch := range tsGoGoarches {
508+ fs .addPathHashTargetLocked (fmt .Sprintf ("tsgo-%s-%s" , goos , goarch ))
509+ }
510+ }
511+
512+ return nil
513+ }
514+
515+ // should return [staleErr] on non-I/O-error-related miss.
516+ func (fs * FS ) handleTargetWithPathHash (ph pathHash ) (handleTarget , error ) {
517+ var zero handleTarget
518+
519+ fs .mu .Lock ()
520+ defer fs .mu .Unlock ()
521+
522+ if err := fs .initHandleMapsLocked (); err != nil {
523+ return zero , err
524+ }
525+
526+ ht , ok := fs .pathHashTarget [ph ]
527+ if ! ok {
528+ return zero , staleErr (fmt .Errorf ("unknown path hash %02x" , ph ))
529+ }
530+ return ht , nil
531+ }
532+
533+ func (fs * FS ) addModuleVersionPathsLocked (mv store.ModuleVersion ) {
534+ // Make "cache/download/github.com/foo/bar/@v/v1.2.3.foo" and then walk up
535+ // its directories to add them to the cache. We don't care about the "foo"
536+ // final component; just the directories above it.
537+ fs .addParentPathsLocked (cdPath (mv , "foo" ))
538+
539+ // Now add the zip root directories above the one with the "@" in it.
540+ // e.g. "github.com/!azure/azure-sdk-for-go/sdk" components above
541+ // "github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.11.0"
542+ fs .addParentPathsLocked (zipRoot (mv ))
543+ }
544+
545+ func (fs * FS ) addParentPathsLocked (s string ) {
546+ for {
547+ s = filepath .Dir (s )
548+ switch s {
549+ case "." , "cache" , "cache/download" :
550+ // Stop once you see any of these.
551+ // They're all well-known already, so we're done.
552+ return
553+ }
554+ fs .addPathHashTargetLocked (s )
555+ }
556+ }
557+
558+ func (fs * FS ) addPathHashTargetLocked (path string ) {
559+ h := mkPathHash (path )
560+ ht := handleTarget {path : path }
561+ if path != "" {
562+ ht .segs = strings .Split (path , "/" )
563+ }
564+ fs .pathHashTarget [h ] = ht
565+ }
566+
377567type moduleNameNode struct {
378568 fs.Inode
379569 paths []string
@@ -704,6 +894,20 @@ func (n *pathUnderZipRoot) Read(ctx context.Context, h fs.FileHandle, dest []byt
704894 return fuse .ReadResultData (n .fileContent [int (off ):end ]), 0
705895}
706896
897+ // cdPath returns the "cache/download/go4.org/mem/@v/v0.0.0-20240501181205-ae6ca9944745.foo" file.
898+ // It returns an invalid path if mv is invalid or malformed.
899+ func cdPath (mv store.ModuleVersion , ext string ) string {
900+ escMod , _ := module .EscapePath (mv .Module )
901+ escVer , _ := module .EscapeVersion (mv .Version )
902+ return "cache/download/" + escMod + "/@v/" + escVer + "." + ext
903+ }
904+
905+ func zipRoot (mv store.ModuleVersion ) string {
906+ escMod , _ := module .EscapePath (mv .Module )
907+ escVer , _ := module .EscapeVersion (mv .Version )
908+ return escMod + "@v" + escVer
909+ }
910+
707911type memFile struct {
708912 fs.Inode
709913 contents []byte
0 commit comments