Browse Source

Make fs scan asynchronous

pull/1/head
Tovi Jaeschke-Rogers 3 years ago
parent
commit
35075c741e
3 changed files with 33 additions and 24 deletions
  1. +22
    -16
      Client/Filesystem/FilesystemDiff.go
  2. +6
    -6
      Client/Filesystem/PickFiles.go
  3. +5
    -2
      Variables/Variables.go

+ 22
- 16
Client/Filesystem/FilesystemDiff.go View File

@ -1,13 +1,15 @@
package Filesystem package Filesystem
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"github.com/vbauerster/mpb" "github.com/vbauerster/mpb"
"github.com/zenthangplus/goccm"
bolt "go.etcd.io/bbolt" bolt "go.etcd.io/bbolt"
"golang.org/x/sync/semaphore"
"PackageManager/Client/Database" "PackageManager/Client/Database"
"PackageManager/Client/ProgressBar" "PackageManager/Client/ProgressBar"
@ -89,7 +91,7 @@ func GetFilesystemLength(root string) (int, error) {
return fsCount, e return fsCount, e
} }
func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar, c goccm.ConcurrencyManager) {
func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucket, p string, bar *mpb.Bar) {
var ( var (
newFileObject FileObject newFileObject FileObject
knownFileObject FileObject knownFileObject FileObject
@ -99,7 +101,6 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke
defer func() { defer func() {
bar.Increment() bar.Increment()
c.Done()
}() }()
pick = picksBucket.Get([]byte(p)) pick = picksBucket.Get([]byte(p))
@ -136,22 +137,20 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke
func GetFilesystemDiff(root string) (FilesystemStatus, error) { func GetFilesystemDiff(root string) (FilesystemStatus, error) {
var ( var (
fsStatus FilesystemStatus = FilesystemStatus{}
rootStat os.FileInfo
fsStatus FilesystemStatus = FilesystemStatus{}
sem *semaphore.Weighted
picksBucket *bolt.Bucket picksBucket *bolt.Bucket
indexBucket *bolt.Bucket indexBucket *bolt.Bucket
bar *mpb.Bar
fsCount int
c goccm.ConcurrencyManager
e error
rootStat os.FileInfo
bar *mpb.Bar
fsCount int
poolSize int
e error
) )
poolSize = runtime.NumCPU()
sem = semaphore.NewWeighted(int64(poolSize))
rootStat, e = os.Stat(root) rootStat, e = os.Stat(root)
if e != nil { if e != nil {
return fsStatus, e return fsStatus, e
@ -189,7 +188,13 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) {
return nil return nil
} }
go fsStatus.parseFile(indexBucket, picksBucket, p, bar, c)
Variables.WG.Add(1)
sem.Acquire(context.Background(), 1)
go func() {
fsStatus.parseFile(indexBucket, picksBucket, p, bar)
Variables.WG.Done()
sem.Release(1)
}()
return nil return nil
}) })
@ -202,6 +207,7 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) {
return nil return nil
}) })
Variables.WG.Wait()
ProgressBar.CloseBar(bar) ProgressBar.CloseBar(bar)
return nil return nil


+ 6
- 6
Client/Filesystem/PickFiles.go View File

@ -63,7 +63,7 @@ func pickFilesRecursive(rootPath string) error {
bar.Increment() bar.Increment()
e = AddFileToBucket(picksBucket, f) e = AddFileToBucket(picksBucket, f)
if e != nil { if e != nil {
return nil
return e
} }
} }
} }
@ -73,7 +73,7 @@ func pickFilesRecursive(rootPath string) error {
bar.Increment() bar.Increment()
e = AddFileToBucket(picksBucket, f) e = AddFileToBucket(picksBucket, f)
if e != nil { if e != nil {
return nil
return e
} }
} }
} }
@ -83,20 +83,20 @@ func pickFilesRecursive(rootPath string) error {
bar.Increment() bar.Increment()
e = RemoveFileFromBucket(indexBucket, f) e = RemoveFileFromBucket(indexBucket, f)
if e != nil { if e != nil {
return nil
return e
} }
e = RemoveFileFromBucket(picksBucket, f) e = RemoveFileFromBucket(picksBucket, f)
if e != nil { if e != nil {
return nil
return e
} }
} }
} }
Variables.WG.Wait()
ProgressBar.CloseBar(bar)
return nil return nil
}) })
ProgressBar.CloseBar(bar)
return e return e
} }


+ 5
- 2
Variables/Variables.go View File

@ -31,10 +31,12 @@ var (
"^/home$", "^/home$",
"^/build$", "^/build$",
"^/tools$", "^/tools$",
"^/opt$",
"^/run/user$", "^/run/user$",
"^/usr/share/zoneinfo-leaps$",
"^/usr/share/zoneinfo$",
"^/usr/share/zsh$", "^/usr/share/zsh$",
"^/usr/share/texmf-dist$",
"^/usr/share/zoneinfo$",
"^/usr/share/zoneinfo-leaps$",
"^/tmp$", "^/tmp$",
"^/var/db$", "^/var/db$",
"^/var/cache$", "^/var/cache$",
@ -59,6 +61,7 @@ var (
"^/var/lib/mlocate/mlocate.db$", "^/var/lib/mlocate/mlocate.db$",
"^/var/lib/krb5kdc/kdc.conf$", "^/var/lib/krb5kdc/kdc.conf$",
"^/var/lib/alsa/asound.state$", "^/var/lib/alsa/asound.state$",
"^/run/systemd/journal/kernel-seqnum$",
} }
) )


Loading…
Cancel
Save