From 35075c741eacea403828fb4a446ff5402f56fa75 Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Mon, 19 Jul 2021 17:54:39 +0930 Subject: [PATCH] Make fs scan asynchronous --- Client/Filesystem/FilesystemDiff.go | 38 +++++++++++++++++------------ Client/Filesystem/PickFiles.go | 12 ++++----- Variables/Variables.go | 7 ++++-- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go index 73eb41f..64550de 100644 --- a/Client/Filesystem/FilesystemDiff.go +++ b/Client/Filesystem/FilesystemDiff.go @@ -1,13 +1,15 @@ package Filesystem import ( + "context" "fmt" "os" "path/filepath" + "runtime" "github.com/vbauerster/mpb" - "github.com/zenthangplus/goccm" bolt "go.etcd.io/bbolt" + "golang.org/x/sync/semaphore" "PackageManager/Client/Database" "PackageManager/Client/ProgressBar" @@ -89,7 +91,7 @@ func GetFilesystemLength(root string) (int, error) { return fsCount, e } -func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar, c goccm.ConcurrencyManager) { +func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) { var ( newFileObject FileObject knownFileObject FileObject @@ -99,7 +101,6 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke defer func() { bar.Increment() - c.Done() }() pick = picksBucket.Get([]byte(p)) @@ -136,22 +137,20 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke func GetFilesystemDiff(root string) (FilesystemStatus, error) { var ( - fsStatus FilesystemStatus = FilesystemStatus{} - - rootStat os.FileInfo - + fsStatus FilesystemStatus = FilesystemStatus{} + sem *semaphore.Weighted picksBucket *bolt.Bucket indexBucket *bolt.Bucket - - bar *mpb.Bar - - fsCount int - - c goccm.ConcurrencyManager - - e error + rootStat os.FileInfo + bar *mpb.Bar + fsCount int + poolSize int + e error ) + poolSize = runtime.NumCPU() + sem = semaphore.NewWeighted(int64(poolSize)) + rootStat, e = os.Stat(root) if e != nil { return fsStatus, e @@ -189,7 +188,13 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil } - go fsStatus.parseFile(indexBucket, picksBucket, p, bar, c) + Variables.WG.Add(1) + sem.Acquire(context.Background(), 1) + go func() { + fsStatus.parseFile(indexBucket, picksBucket, p, bar) + Variables.WG.Done() + sem.Release(1) + }() return nil }) @@ -202,6 +207,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil }) + Variables.WG.Wait() ProgressBar.CloseBar(bar) return nil diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go index 984b7a7..330cf4a 100644 --- a/Client/Filesystem/PickFiles.go +++ b/Client/Filesystem/PickFiles.go @@ -63,7 +63,7 @@ func pickFilesRecursive(rootPath string) error { bar.Increment() e = AddFileToBucket(picksBucket, f) if e != nil { - return nil + return e } } } @@ -73,7 +73,7 @@ func pickFilesRecursive(rootPath string) error { bar.Increment() e = AddFileToBucket(picksBucket, f) if e != nil { - return nil + return e } } } @@ -83,20 +83,20 @@ func pickFilesRecursive(rootPath string) error { bar.Increment() e = RemoveFileFromBucket(indexBucket, f) if e != nil { - return nil + return e } e = RemoveFileFromBucket(picksBucket, f) if e != nil { - return nil + return e } } } + Variables.WG.Wait() + ProgressBar.CloseBar(bar) return nil }) - ProgressBar.CloseBar(bar) - return e } diff --git a/Variables/Variables.go b/Variables/Variables.go index 15482d8..8ab00a0 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -31,10 +31,12 @@ var ( "^/home$", "^/build$", "^/tools$", + "^/opt$", "^/run/user$", - "^/usr/share/zoneinfo-leaps$", - "^/usr/share/zoneinfo$", "^/usr/share/zsh$", + "^/usr/share/texmf-dist$", + "^/usr/share/zoneinfo$", + "^/usr/share/zoneinfo-leaps$", "^/tmp$", "^/var/db$", "^/var/cache$", @@ -59,6 +61,7 @@ var ( "^/var/lib/mlocate/mlocate.db$", "^/var/lib/krb5kdc/kdc.conf$", "^/var/lib/alsa/asound.state$", + "^/run/systemd/journal/kernel-seqnum$", } )