From 34f5b0c9c0d02dabc6a3f2943efda3e3223f80de Mon Sep 17 00:00:00 2001 From: Tovi Jaeschke-Rogers Date: Tue, 20 Jul 2021 11:39:15 +0930 Subject: [PATCH] Add ability to create packages from diff --- Client/Filesystem/CommitFiles.go | 20 ++- Client/Filesystem/FilesystemDiff.go | 18 +- Client/Filesystem/PickFiles.go | 1 - Client/Filesystem/Print.go | 15 +- Client/Package/CreatePackage.go | 262 +++++++++++++++------------- Client/main.go | 33 ++-- Variables/Variables.go | 9 +- 7 files changed, 208 insertions(+), 150 deletions(-) diff --git a/Client/Filesystem/CommitFiles.go b/Client/Filesystem/CommitFiles.go index 8cb066b..5db8987 100644 --- a/Client/Filesystem/CommitFiles.go +++ b/Client/Filesystem/CommitFiles.go @@ -11,24 +11,26 @@ import ( func CommitFiles() error { var ( - fsStatus FilesystemStatus indexBucket *bolt.Bucket + picksBucket *bolt.Bucket bar *mpb.Bar + pickedFiles []string f string e error ) - fsStatus, e = GetFilesystemDiff(Variables.RootDir) - if e != nil { - return e - } - e = Database.FsDB.Batch(func(tx *bolt.Tx) error { indexBucket = tx.Bucket(Variables.FsHashIndexBucket) + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + + picksBucket.ForEach(func(k, v []byte) error { + pickedFiles = append(pickedFiles, string(k)) + return nil + }) - if len(fsStatus.PickedFiles) > 0 { - bar = ProgressBar.InitBar("Commiting...", len(fsStatus.PickedFiles)) - for _, f = range fsStatus.PickedFiles { + if len(pickedFiles) > 0 { + bar = ProgressBar.InitBar("Commiting...", len(pickedFiles)) + for _, f = range pickedFiles { bar.Increment() e = AddFileToBucket(indexBucket, f) if e != nil { diff --git a/Client/Filesystem/FilesystemDiff.go b/Client/Filesystem/FilesystemDiff.go index 64550de..d5d0ffa 100644 --- a/Client/Filesystem/FilesystemDiff.go +++ b/Client/Filesystem/FilesystemDiff.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "runtime" + "sync" "github.com/vbauerster/mpb" bolt "go.etcd.io/bbolt" @@ -17,6 +18,11 @@ import ( "PackageManager/Variables" ) +var ( + fsStatusWG sync.WaitGroup + FsWalkWG sync.WaitGroup +) + type FilesystemStatus struct { NewFiles []string PickedFiles []string @@ -107,7 +113,10 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke known = indexBucket.Get([]byte(p)) if pick != nil { + fsStatusWG.Wait() + fsStatusWG.Add(1) fsStatus.PickedFiles = append(fsStatus.PickedFiles, p) + fsStatusWG.Done() return } @@ -124,7 +133,10 @@ func (fsStatus *FilesystemStatus) parseFile(indexBucket, picksBucket *bolt.Bucke e = newFileObject.IsDifferent(knownFileObject) if e != nil { + fsStatusWG.Wait() + fsStatusWG.Add(1) fsStatus.ModifiedFiles = append(fsStatus.ModifiedFiles, p) + fsStatusWG.Done() } return @@ -188,14 +200,15 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil } - Variables.WG.Add(1) + FsWalkWG.Add(1) sem.Acquire(context.Background(), 1) go func() { fsStatus.parseFile(indexBucket, picksBucket, p, bar) - Variables.WG.Done() sem.Release(1) + FsWalkWG.Done() }() + FsWalkWG.Wait() return nil }) @@ -207,7 +220,6 @@ func GetFilesystemDiff(root string) (FilesystemStatus, error) { return nil }) - Variables.WG.Wait() ProgressBar.CloseBar(bar) return nil diff --git a/Client/Filesystem/PickFiles.go b/Client/Filesystem/PickFiles.go index 330cf4a..f41ab86 100644 --- a/Client/Filesystem/PickFiles.go +++ b/Client/Filesystem/PickFiles.go @@ -92,7 +92,6 @@ func pickFilesRecursive(rootPath string) error { } } - Variables.WG.Wait() ProgressBar.CloseBar(bar) return nil }) diff --git a/Client/Filesystem/Print.go b/Client/Filesystem/Print.go index 668cb61..172a200 100644 --- a/Client/Filesystem/Print.go +++ b/Client/Filesystem/Print.go @@ -5,9 +5,16 @@ import ( "fmt" ) -func PrintFiles(files []string, color func(...interface{}) string) { - var f string - for _, f = range files { +func PrintFiles(files []string, color func(...interface{}) string, showInt bool) { + var ( + f string + i int + ) + for i, f = range files { + if showInt { + fmt.Printf("\t%d - %s\n", i, color(f)) + continue + } fmt.Printf("\t%s\n", color(f)) } } @@ -18,7 +25,7 @@ func PrintFilesLength(files []string) { func PrintFilesOrLength(files []string, color func(...interface{}) string) { if (Variables.VerboseOutput && len(files) != 0) || (len(files) < 25 && len(files) > 0) { - PrintFiles(files, color) + PrintFiles(files, color, false) return } PrintFilesLength(files) diff --git a/Client/Package/CreatePackage.go b/Client/Package/CreatePackage.go index 6da7a3a..09e9ed9 100644 --- a/Client/Package/CreatePackage.go +++ b/Client/Package/CreatePackage.go @@ -1,134 +1,154 @@ package Package +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + bolt "go.etcd.io/bbolt" + + "PackageManager/Archive" + "PackageManager/Client/Database" + "PackageManager/Client/Filesystem" + "PackageManager/Color" + "PackageManager/Helper" + "PackageManager/Variables" +) + +func writeManifestFile(path, name, version string) error { + var ( + cmd *exec.Cmd + manifest string + filePath string + e error + ) + + manifest = fmt.Sprintf(`name: "%s" +version: "%s" +dependancies: + - pkg1: v1.0.0 + - pkg2: v1.0.0 +`, name, version) + + filePath = filepath.Join(path, "manifest.yml") + + e = ioutil.WriteFile(filePath, []byte(manifest), 0644) + if e != nil { + return e + } + + cmd = exec.Command(Variables.Editor, filePath) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + return cmd.Run() +} + func CreatePackage() error { - return nil + var ( + picksBucket *bolt.Bucket + pickedFiles []string + pkgFiles []string + choices string + choicesSplit []string + pkgName string + pkgVersion string + pkgNameVersion string + tmpDir string + index int + e error + ) + + fmt.Println("Initialising package creation...") + + e = Database.FsDB.View(func(tx *bolt.Tx) error { + picksBucket = tx.Bucket(Variables.FsHashPicksBucket) + + picksBucket.ForEach(func(k, v []byte) error { + pickedFiles = append(pickedFiles, string(k)) + return nil + }) - /* - var ( - dirtyFiles map[int]string - newFiles map[int]string - pkgFiles map[int]string = make(map[int]string) - choices string - choicesSplit []string - filePath string - pkgName string - pkgVersion string - pkgNameVersion string - tmpDir string - index int - ok bool - e error - ) - - fmt.Println("Initialising package creation...") - - dirtyFiles, newFiles, e = Filesystem.GetFilesystemDiff() - if e != nil { - return e - } + return nil + }) - fmt.Println("\nModified files...") - for i, file := range dirtyFiles { - fmt.Printf( - "\t%d - %s\n", - i, - Color.Red(file), - ) - } + fmt.Println("Added files:") + Filesystem.PrintFiles(pickedFiles, Color.Green, true) - fmt.Println("\nNew files...") - for i, file := range newFiles { - fmt.Printf( - "\t%d - %s\n", - i, - Color.Red(file), - ) - } + fmt.Println("Please select the files you would like to use to create the package. Leave empty for all.") + choices = Helper.Input() - fmt.Println("Please select the files you would like to use to create the package. Leave empty for all.") - choices = Helper.Input() + if choices == "" { + pkgFiles = pickedFiles + } else { + choicesSplit = strings.Split(choices, ",") - if choices == "" { - for i, file := range dirtyFiles { - pkgFiles[i] = file + for _, i := range choicesSplit { + index, e = strconv.Atoi(i) + if e != nil { + // TODO: Handle this error + panic(e) } - for i, file := range newFiles { - pkgFiles[i] = file + if len(pickedFiles) < index { + return errors.New("Invalid choice") } - } else { - - choicesSplit = strings.Split(choices, ",") - - for _, i := range choicesSplit { - index, e = strconv.Atoi(i) - if e != nil { - // TODO: Handle this error - panic(e) - } - filePath, ok = dirtyFiles[index] - if !ok { - filePath, ok = newFiles[index] - if !ok { - return errors.New("Invalid package selection") - } - } - pkgFiles[index] = filePath - } - } - - fmt.Println("Please enter the package name:") - pkgName = Helper.Input() - if pkgName == "" { - return errors.New("Invalid package name") - } - - fmt.Println("Please enter the package version:") - pkgVersion = Helper.Input() - if pkgVersion == "" { - return errors.New("Invalid package name") - } - - fmt.Printf("Package Name: %s\n", pkgName) - fmt.Printf("Package Version: %s\n", pkgVersion) - - fmt.Println("Files to be added") - for i, file := range pkgFiles { - fmt.Printf( - "\t%d - %s\n", - i, - Color.Green(file), - ) - } - - fmt.Println("Is this correct? [y/N]") - if strings.ToLower(Helper.Input()) != "y" { - return errors.New("User aborted") - } - - pkgNameVersion = fmt.Sprintf("%s-%s", pkgName, pkgVersion) - - tmpDir, e = ioutil.TempDir("/tmp", pkgNameVersion) - if e != nil { - return e - } - defer os.RemoveAll(tmpDir) - - for _, file := range pkgFiles { - Filesystem.CopyFile(file, filepath.Join(tmpDir, file)) - } - - // TODO: Add dependancy management here - - e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz") - if e != nil { - return e + pkgFiles = append(pkgFiles, pickedFiles[index]) } + } + + fmt.Println("Please enter the package name:") + pkgName = Helper.Input() + if pkgName == "" { + return errors.New("Invalid package name") + } + + fmt.Println("Please enter the package version:") + pkgVersion = Helper.Input() + if pkgVersion == "" { + return errors.New("Invalid package name") + } + + fmt.Printf("Package Name: %s\n", pkgName) + fmt.Printf("Package Version: %s\n", pkgVersion) + + fmt.Println("Files to be added") + Filesystem.PrintFiles(pkgFiles, Color.Green, false) + + fmt.Println("Is this correct? [y/N]") + if strings.ToLower(Helper.Input()) != "y" { + return errors.New("User aborted") + } + + pkgNameVersion = fmt.Sprintf("%s-%s", pkgName, pkgVersion) + + tmpDir, e = ioutil.TempDir("/tmp", pkgNameVersion) + if e != nil { + return e + } + defer os.RemoveAll(tmpDir) + + for _, file := range pkgFiles { + Filesystem.CopyFile(file, filepath.Join(tmpDir, file)) + } + + e = writeManifestFile(tmpDir, pkgName, pkgVersion) + if e != nil { + return e + } + + e = Archive.TarGzip(tmpDir, pkgNameVersion+".tar.gz") + if e != nil { + return e + } + + fmt.Printf( + Color.Green("\nSuccessfully created package %s\n"), + pkgNameVersion, + ) - fmt.Printf( - Color.Green("\nSuccessfully created package %s\n"), - pkgNameVersion, - ) - - return nil - */ + return nil } diff --git a/Client/main.go b/Client/main.go index f820c15..1797c66 100644 --- a/Client/main.go +++ b/Client/main.go @@ -7,6 +7,7 @@ import ( "PackageManager/Client/Database" "PackageManager/Client/Filesystem" + "PackageManager/Client/Package" "PackageManager/Color" "PackageManager/Helper" "PackageManager/Variables" @@ -16,6 +17,11 @@ func HelpMsg() { var helpMsg string helpMsg = `Usage of %s: +General: + -V | -verbose + Verbose output + +Filesystem diff: -Af | -add-files Add files @@ -27,9 +33,9 @@ func HelpMsg() { -Rf | -reset Reset added files - - -V | -verbose - Verbose output + + -Cp | -create-pkg + Create package from fs diff ` helpMsg = fmt.Sprintf(helpMsg, os.Args[0]) fmt.Println(helpMsg) @@ -46,6 +52,9 @@ func main() { resetAddedFilesFlag bool resetAddedFilesFlagLong bool + createPackageFlag bool + createPackageFlagLong bool + verboseOutputFlag bool verboseOutputFlagLong bool @@ -83,6 +92,9 @@ func main() { flag.BoolVar(&resetAddedFilesFlag, "Rf", false, "Reset added files") flag.BoolVar(&resetAddedFilesFlagLong, "reset", false, "Reset added files") + flag.BoolVar(&createPackageFlag, "Cp", false, "Create package from fs diff") + flag.BoolVar(&createPackageFlagLong, "create-pkg", false, "Create package from fs diff") + flag.Parse() Variables.VerboseOutput = verboseOutputFlag || verboseOutputFlagLong @@ -137,14 +149,15 @@ func main() { return } - /* - if createPackageFlag || createPackageFlagLong { - e = Package.CreatePackage() - if e != nil { - panic(e) - } - return + if createPackageFlag || createPackageFlagLong { + e = Package.CreatePackage() + if e != nil { + panic(e) } + return + } + + /* if installLocalPackageFlag || installLocalPackageFlagLong { e = Package.InstallPackage(flag.Args()) diff --git a/Variables/Variables.go b/Variables/Variables.go index 8ab00a0..bffc376 100644 --- a/Variables/Variables.go +++ b/Variables/Variables.go @@ -2,7 +2,6 @@ package Variables import ( "os" - "sync" ) const ( @@ -11,7 +10,7 @@ const ( ) var ( - WG sync.WaitGroup + Editor string = "vi" VerboseOutput bool = false RootDir string = "/" @@ -68,9 +67,15 @@ var ( func init() { var ( rootDir string + editor string ) rootDir = os.Getenv("ROOTDIR") if rootDir != "" { RootDir = rootDir } + editor = os.Getenv("EDITOR") + if editor != "" { + Editor = editor + } + }