|
|
|
|
@ -55,8 +55,6 @@ func getSyncCommand(repo *config.Repo) (cmds [][]string) { |
|
|
|
|
return append(cmds, cscSyncDebianStep1(repo), cscSyncDebianStep2(repo)) |
|
|
|
|
case "csc-sync-debian-cd": |
|
|
|
|
return append(cmds, cscSyncDebianCD(repo)) |
|
|
|
|
case "csc-sync-gentoo": |
|
|
|
|
return append(cmds, cscSyncGentoo(repo)) |
|
|
|
|
case "csc-sync-s3": |
|
|
|
|
return append(cmds, cscSyncS3(repo)) |
|
|
|
|
case "csc-sync-ssh": |
|
|
|
|
@ -216,7 +214,7 @@ func cscSyncCDImage(repo *config.Repo) []string { |
|
|
|
|
|
|
|
|
|
func cscSyncCephStep1(repo *config.Repo) []string { |
|
|
|
|
args := []string{ |
|
|
|
|
"rsync", "--stats", "--progress", |
|
|
|
|
"nice", "rsync", "--stats", "--progress", |
|
|
|
|
repo.RsyncHost + "::ceph", |
|
|
|
|
"--recursive", "--times", "--links", |
|
|
|
|
"--hard-links", |
|
|
|
|
@ -236,7 +234,7 @@ func cscSyncCephStep1(repo *config.Repo) []string { |
|
|
|
|
|
|
|
|
|
func cscSyncCephStep2(repo *config.Repo) []string { |
|
|
|
|
args := []string{ |
|
|
|
|
"&&", "rsync", "--stats", "--progress", |
|
|
|
|
"nice", "rsync", "--stats", "--progress", |
|
|
|
|
repo.RsyncHost + "::ceph", |
|
|
|
|
"--recursive", "--times", "--links", |
|
|
|
|
"--hard-links", "--delete-after", |
|
|
|
|
@ -291,24 +289,17 @@ func cscSyncDebianCD(repo *config.Repo) []string { |
|
|
|
|
return args |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func cscSyncGentoo(repo *config.Repo) []string { |
|
|
|
|
repo.RsyncUser = "gentoo" |
|
|
|
|
repo.PasswordFile = "gentoo-distfiles" |
|
|
|
|
return cscSyncStandard(repo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// TODO: check for special stuff that rcloning S3 needs
|
|
|
|
|
func cscSyncS3(repo *config.Repo) []string { |
|
|
|
|
// could not find a argument for "RCLONE_CONFIG_S3_TYPE=s3"
|
|
|
|
|
// so an adhoc solution was put in spawnProcess (if possible remove)
|
|
|
|
|
args := []string{ |
|
|
|
|
// RsyncHost is just a regular host (https://s3.repo.saltproject.io) in this case
|
|
|
|
|
"RCLONE_CONFIG_S3_ENDPOINT=" + repo.RsyncHost, |
|
|
|
|
"RCLONE_CONFIG_S3_TYPE=s3", |
|
|
|
|
"RCLONE_CONFIG_S3_PROVIDER=Other", |
|
|
|
|
"RCLONE_CONFIG_S3_ENV_AUTH=false", |
|
|
|
|
"nice", "rclone", "sync", |
|
|
|
|
"--fast-list", |
|
|
|
|
"--use-server-modtime", |
|
|
|
|
"--bind", config.Conf.IPv4Address, |
|
|
|
|
"--bind=" + config.Conf.IPv4Address, |
|
|
|
|
"--s3-endpoint=" + repo.RsyncHost, // RCLONE_CONFIG_S3_ENDPOINT=$2
|
|
|
|
|
"--s3-provider=Other", // RCLONE_CONFIG_S3_PROVIDER=Other
|
|
|
|
|
"--s3-env-auth=false", // RCLONE_CONFIG_S3_ENV_AUTH=false
|
|
|
|
|
"s3:s3/", |
|
|
|
|
} |
|
|
|
|
args = append(args, buildDownloadDir(repo)) |
|
|
|
|
@ -322,8 +313,7 @@ func cscSyncSSH(repo *config.Repo) []string { |
|
|
|
|
"--stats", "-4", |
|
|
|
|
} |
|
|
|
|
args = append(args, addConditionalFlags(repo, stdFlags|delete)...) |
|
|
|
|
// PasswordFile should point to the SSH_KEYFILE
|
|
|
|
|
args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile)) |
|
|
|
|
args = append(args, "-e", fmt.Sprintf("\"ssh -b %s -i %s\"", config.Conf.IPv4Address, repo.PasswordFile)) |
|
|
|
|
args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo)) |
|
|
|
|
|
|
|
|
|
return args |
|
|
|
|
@ -372,12 +362,10 @@ func cscSyncWget(repo *config.Repo) []string { |
|
|
|
|
"--no-host-directories", |
|
|
|
|
"--cut-dirs=1", |
|
|
|
|
"--content-disposition", |
|
|
|
|
"--execute", |
|
|
|
|
"robots=off", |
|
|
|
|
"--execute", "robots=off", |
|
|
|
|
"--recursive", |
|
|
|
|
"--reject", "\"*\\?*\"", |
|
|
|
|
"--directory-prefix=" + buildDownloadDir(repo), |
|
|
|
|
// RsyncHost is just a regular host (https://mirror.racket-lang.org/installers/) in this case
|
|
|
|
|
repo.RsyncHost, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
Andrew Wang
8 months ago