Skip to content

Commit

Permalink
Merge pull request #272 from SAP/copy_dir_fix
Browse files Browse the repository at this point in the history 
 added flag parallel to assemble command
  • Loading branch information
@ShimiT
ShimiT authored Feb 27, 2019
2 parents 425bdbd + e6ade5d commit d48ccc0
Show file tree
Hide file tree
Showing 6 changed files with 127 additions and 42 deletions.
15 changes: 12 additions & 3 deletions cmd/assembly.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
var assembleCmdSrc string
var assembleCmdTrg string
var assembleCmdMtarName string
var assembleCmdParallel string

func init() {
assemblyCommand.Flags().StringVarP(&assembleCmdSrc,
Expand All @@ -27,6 +28,9 @@ func init() {
"target", "t", "", "the path to the MBT results folder; the current path is default")
assemblyCommand.Flags().StringVarP(&assembleCmdMtarName,
"mtar", "m", "", "the archive name")
assemblyCommand.Flags().StringVarP(&assembleCmdParallel,
"parallel", "p", "false", "if true content copying will run in parallel")

}

// Generate mtar from build artifacts
Expand All @@ -37,18 +41,23 @@ var assemblyCommand = &cobra.Command{
ValidArgs: []string{"Deployment descriptor location"},
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
err := assembly(assembleCmdSrc, assembleCmdTrg, defaultPlatform, mtarCmdMtarName, os.Getwd)
err := assembly(assembleCmdSrc, assembleCmdTrg, defaultPlatform, mtarCmdMtarName, assembleCmdParallel, os.Getwd)
logError(err)
return err
},
SilenceUsage: true,
SilenceErrors: true,
}

func assembly(source, target, platform, mtarName string, getWd func() (string, error)) error {
func assembly(source, target, platform, mtarName, copyInParallel string, getWd func() (string, error)) error {
logs.Logger.Info("assembling the MTA project...")

parallelCopy, err := strconv.ParseBool(copyInParallel)
if err != nil {
parallelCopy = false
}
// copy from source to target
err := artifacts.CopyMtaContent(source, target, dir.Dep, getWd)
err = artifacts.CopyMtaContent(source, target, dir.Dep, parallelCopy, getWd)
if err != nil {
return errors.Wrap(err, "assembly of the MTA project failed when copying the MTA content")
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/assembly_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ var _ = Describe("Assembly", func() {
})
It("Sanity", func() {
err := assembly(getTestPath("assembly-sample"),
getTestPath("result"), "cf", "", os.Getwd)
getTestPath("result"), "cf", "", "?", os.Getwd)
Ω(err).Should(Succeed())
Ω(getTestPath("result", "com.sap.xs2.samples.javahelloworld_0.1.0.mtar")).Should(BeAnExistingFile())
})
var _ = DescribeTable("Fails on location initialization", func(maxCalls int) {
calls := 0
err := assembly("",
getTestPath("result"), "cf", "", func() (string, error) {
getTestPath("result"), "cf", "", "true", func() (string, error) {
calls++
if calls >= maxCalls {
return "", errors.New("error")
Expand Down
41 changes: 36 additions & 5 deletions internal/archive/fsops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func CreateFile(path string) (file *os.File, err error) {
}

// CopyDir - copy directory content
func CopyDir(src string, dst string, withParents bool) error {
func CopyDir(src string, dst string, withParents bool, copyDirEntries func(entries []os.FileInfo, src, dst string) error) error {
src = filepath.Clean(src)
dst = filepath.Clean(dst)

Expand All @@ -158,7 +158,7 @@ func CopyDir(src string, dst string, withParents bool) error {
if err != nil {
return err
}
return CopyEntries(entries, src, dst)
return copyDirEntries(entries, src, dst)
}

// CopyByPatterns - copy files/directories according to patterns
Expand Down Expand Up @@ -231,7 +231,7 @@ func copyEntries(entries []string, source, target, pattern string) error {
}
targetEntry := filepath.Join(target, filepath.Base(entry))
if info.IsDir() {
err = CopyDir(entry, targetEntry, true)
err = CopyDir(entry, targetEntry, true, CopyEntries)
} else {
err = CopyFileWithMode(entry, targetEntry, info.Mode())
}
Expand All @@ -245,7 +245,38 @@ func copyEntries(entries []string, source, target, pattern string) error {
}

// CopyEntries - copies entries (files and directories) from source to destination folder
func CopyEntries(entries []os.FileInfo, src, dst string) (rerr error) {
func CopyEntries(entries []os.FileInfo, src, dst string) error {

if len(entries) == 0 {
return nil
}
for _, entry := range entries {
var err error
srcPath := filepath.Join(src, entry.Name())
dstPath := filepath.Join(dst, entry.Name())

if entry.IsDir() {
// execute recursively
err = CopyDir(srcPath, dstPath, false, CopyEntries)
} else {
// Todo check posix compatibility
if entry.Mode()&os.ModeSymlink != 0 {
logs.Logger.Infof(
`copying of the entries from the "%v" folder to the "%v" folder skipped the "%v" entry because its mode is a symbolic link`,
src, dst, entry.Name())
} else {
err = CopyFileWithMode(srcPath, dstPath, entry.Mode())
}
}
if err != nil {
return err
}
}
return nil
}

// CopyEntriesInParallel - copies entries (files and directories) from source to destination folder in parallel
func CopyEntriesInParallel(entries []os.FileInfo, src, dst string) (rerr error) {

if len(entries) == 0 {
return nil
Expand All @@ -266,7 +297,7 @@ func CopyEntries(entries []os.FileInfo, src, dst string) (rerr error) {

if e.IsDir() {
// execute recursively
err = CopyDir(srcPath, dstPath, false)
err = CopyDir(srcPath, dstPath, false, CopyEntriesInParallel)
} else {
// Todo check posix compatibility
if e.Mode()&os.ModeSymlink != 0 {
Expand Down
44 changes: 40 additions & 4 deletions internal/archive/fsops_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,21 +109,42 @@ var _ = Describe("FSOPS", func() {
os.RemoveAll(targetPath)
})

It("Sanity", func() {
It("Sanity - parallel", func() {
sourcePath := getFullPath("testdata", "level2")
Ω(CopyDir(sourcePath, targetPath, true, CopyEntriesInParallel)).Should(Succeed())
Ω(countFilesInDir(targetPath)).Should(Equal(countFilesInDir(sourcePath)))
})

It("Sanity - not parallel", func() {
sourcePath := getFullPath("testdata", "level2")
Ω(CopyDir(sourcePath, targetPath, true)).Should(Succeed())
Ω(CopyDir(sourcePath, targetPath, true, CopyEntries)).Should(Succeed())
Ω(countFilesInDir(targetPath)).Should(Equal(countFilesInDir(sourcePath)))
})

It("TargetFileLocked", func() {
f, _ := os.Create(targetPath)
sourcePath := getFullPath("testdata", "level2")
Ω(CopyDir(sourcePath, targetPath, true)).Should(HaveOccurred())
Ω(CopyDir(sourcePath, targetPath, true, CopyEntries)).Should(HaveOccurred())
f.Close()
})

It("TargetFileLocked", func() {
f, _ := os.Create(targetPath)
sourcePath := getFullPath("testdata", "level2")
Ω(CopyDir(sourcePath, targetPath, true, CopyEntriesInParallel)).Should(HaveOccurred())
f.Close()
})

var _ = DescribeTable("Invalid cases", func(source, target string) {
Ω(CopyDir(source, targetPath, true)).Should(HaveOccurred())
Ω(CopyDir(source, targetPath, true, CopyEntries)).Should(HaveOccurred())
},
Entry("SourceDirectoryDoesNotExist", getFullPath("testdata", "level5"), targetPath),
Entry("SourceIsNotDirectory", getFullPath("testdata", "level2", "level2_one.txt"), targetPath),
Entry("DstDirectoryNotValid", getFullPath("level2"), ":"),
)

var _ = DescribeTable("Invalid cases - parallel", func(source, target string) {
Ω(CopyDir(source, targetPath, true, CopyEntriesInParallel)).Should(HaveOccurred())
},
Entry("SourceDirectoryDoesNotExist", getFullPath("testdata", "level5"), targetPath),
Entry("SourceIsNotDirectory", getFullPath("testdata", "level2", "level2_one.txt"), targetPath),
Expand Down Expand Up @@ -168,6 +189,21 @@ var _ = Describe("FSOPS", func() {
Ω(countFilesInDir(sourcePath) - 1).Should(Equal(countFilesInDir(targetPath)))
os.RemoveAll(targetPath)
})
It("Sanity - copy in parallel", func() {
sourcePath := getFullPath("testdata", "level2", "level3")
targetPath := getFullPath("testdata", "result")
os.MkdirAll(targetPath, os.ModePerm)
files, _ := ioutil.ReadDir(sourcePath)
// Files wrapped to overwrite their methods
var filesWrapped []os.FileInfo
Ω(CopyEntriesInParallel(filesWrapped, sourcePath, targetPath)).Should(Succeed())
for _, file := range files {
filesWrapped = append(filesWrapped, testFile{file: file})
}
Ω(CopyEntriesInParallel(filesWrapped, sourcePath, targetPath)).Should(Succeed())
Ω(countFilesInDir(sourcePath) - 1).Should(Equal(countFilesInDir(targetPath)))
os.RemoveAll(targetPath)
})
})

var _ = Describe("Copy By Patterns", func() {
Expand Down
31 changes: 17 additions & 14 deletions internal/artifacts/module_arch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func copyModuleArchive(ep dir.IModule, modulePath, moduleName string) error {

// CopyMtaContent copies the content of all modules and resources which are presented in the deployment descriptor,
// in the source directory, to the target directory
func CopyMtaContent(source, target, desc string, wdGetter func() (string, error)) error {
func CopyMtaContent(source, target, desc string, copyInParallel bool, wdGetter func() (string, error)) error {

logs.Logger.Info("copying the MTA content...")
loc, err := dir.Location(source, target, desc, wdGetter)
Expand All @@ -167,29 +167,29 @@ func CopyMtaContent(source, target, desc string, wdGetter func() (string, error)
if err != nil {
return errors.Wrapf(err, `copying the MTA content failed when parsing the %s file`, loc.GetMtaYamlPath())
}
err = copyModuleContent(loc.GetSource(), loc.GetTargetTmpDir(), mta)
err = copyModuleContent(loc.GetSource(), loc.GetTargetTmpDir(), mta, copyInParallel)
if err != nil {
return err
}

err = copyRequiredDependencyContent(loc.GetSource(), loc.GetTargetTmpDir(), mta)
err = copyRequiredDependencyContent(loc.GetSource(), loc.GetTargetTmpDir(), mta, copyInParallel)
if err != nil {
return err
}

return copyResourceContent(loc.GetSource(), loc.GetTargetTmpDir(), mta)
return copyResourceContent(loc.GetSource(), loc.GetTargetTmpDir(), mta, copyInParallel)
}

func copyModuleContent(source, target string, mta *mta.MTA) error {
return copyMtaContent(source, target, getModulesWithPaths(mta.Modules))
func copyModuleContent(source, target string, mta *mta.MTA, copyInParallel bool) error {
return copyMtaContent(source, target, getModulesWithPaths(mta.Modules), copyInParallel)
}

func copyResourceContent(source, target string, mta *mta.MTA) error {
return copyMtaContent(source, target, getResourcesPaths(mta.Resources))
func copyResourceContent(source, target string, mta *mta.MTA, copyInParallel bool) error {
return copyMtaContent(source, target, getResourcesPaths(mta.Resources), copyInParallel)
}

func copyRequiredDependencyContent(source, target string, mta *mta.MTA) error {
return copyMtaContent(source, target, getRequiredDependencyPaths(mta.Modules))
func copyRequiredDependencyContent(source, target string, mta *mta.MTA, copyInParallel bool) error {
return copyMtaContent(source, target, getRequiredDependencyPaths(mta.Modules), copyInParallel)
}

func getRequiredDependencyPaths(mtaModules []*mta.Module) []string {
Expand All @@ -210,7 +210,7 @@ func getRequiredDependenciesWithPathsForModule(module *mta.Module) []string {
}
return result
}
func copyMtaContent(source, target string, mtaPaths []string) error {
func copyMtaContent(source, target string, mtaPaths []string, copyInParallel bool) error {
copiedMtaContents := make([]string, 0)
for _, mtaPath := range mtaPaths {
sourceMtaContent := filepath.Join(source, mtaPath)
Expand All @@ -220,7 +220,7 @@ func copyMtaContent(source, target string, mtaPaths []string) error {
}
copiedMtaContents = append(copiedMtaContents, mtaPath)
targetMtaContent := filepath.Join(target, mtaPath)
err := copyMtaContentFromPath(sourceMtaContent, targetMtaContent, mtaPath, target)
err := copyMtaContentFromPath(sourceMtaContent, targetMtaContent, mtaPath, target, copyInParallel)
if err != nil {
return handleCopyMtaContentFailure(target, copiedMtaContents,
`error copying the "%s" MTA content to the "%s" target directory because: %s`, []interface{}{mtaPath, source, err.Error()})
Expand All @@ -240,10 +240,13 @@ func handleCopyMtaContentFailure(targetLocation string, copiedMtaContents []stri
return fmt.Errorf(message+"; cleanup failed", messageArguments...)
}

func copyMtaContentFromPath(sourceMtaContent, targetMtaContent, mtaContentPath, target string) error {
func copyMtaContentFromPath(sourceMtaContent, targetMtaContent, mtaContentPath, target string, copyInParallel bool) error {
mtaContentInfo, _ := os.Stat(sourceMtaContent)
if mtaContentInfo.IsDir() {
return dir.CopyDir(sourceMtaContent, targetMtaContent, true)
if copyInParallel {
return dir.CopyDir(sourceMtaContent, targetMtaContent, true, dir.CopyEntriesInParallel)
}
return dir.CopyDir(sourceMtaContent, targetMtaContent, true, dir.CopyEntries)
}

mtaContentParentDir := filepath.Dir(mtaContentPath)
Expand Down
Loading

0 comments on commit d48ccc0

Please sign in to comment.