cloud
/
cloudbuild
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add support for deb822

This commit is contained in:
Max Erenberg 2023-06-18 21:48:42 -04:00
parent 3fa4541152
commit c7f71ce2a9
2 changed files with 208 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

208
pkg/distros/debian_utils.go Normal file
View File

@ -0,0 +1,208 @@
package distros
import (
"fmt"
"net/url"
"strings"
"libguestfs.org/guestfs"
)
func debianCommandToUpdatePackageCache() []string {
return []string{"sudo", "apt", "update"}
}
func (mgr *TemplateManager) replaceDebianMirrorUrl(handle *guestfs.Guestfs, uri string) (string, error) {
parsedUrl, err := url.Parse(uri)
if err != nil {
return "", fmt.Errorf("Could not parse URL '%s': %w", uri, err)
}
// See https://manpages.debian.org/stable/apt/sources.list.5.en.html#URI_SPECIFICATION
// There are many more URI types, but these are the ones which I've seen so far in the
// official Debian cloud images
if parsedUrl.Scheme == "http" || parsedUrl.Scheme == "https" {
parsedUrl.Host = mgr.cfg.MirrorHost
return parsedUrl.String(), nil
} else if parsedUrl.Scheme == "mirror+file" {
// The file is a mirrorlist
filePath := uri[len("mirror+file://"):]
content, err := handle.Cat(filePath)
if err != nil {
return "", fmt.Errorf("Could not read %s: %w", filePath, err)
}
oldLines := strings.Split(content, "\n")
newLines := []string{}
for _, oldLine := range oldLines {
if oldLine == "" || strings.HasPrefix(oldLine, "#") {
newLines = append(newLines, oldLine)
continue
}
parsedUrl, err := url.Parse(oldLine)
if err != nil {
return "", fmt.Errorf("Could not parse URL in %s: '%s'", filePath, oldLine)
}
if parsedUrl.Scheme != "http" && parsedUrl.Scheme != "https" {
return "", fmt.Errorf("Unexpected URL scheme in %s: '%s'", filePath, oldLine)
}
parsedUrl.Host = mgr.cfg.MirrorHost
newLine := parsedUrl.String()
mgr.logger.Debug().
Str("file", filePath).
Str("oldURL", oldLine).
Str("newURL", newLine).
Msg("Replacing URL")
newLines = append(newLines, newLine)
}
newContent := strings.Join(newLines, "\n")
err = handle.Write(filePath, []byte(newContent))
if err != nil {
return "", fmt.Errorf("Error writing to %s: %w", filePath, err)
}
// Original URL (mirror+file://) does not change
return uri, nil
} else {
return "", fmt.Errorf("Unexpected scheme in URL: %s", uri)
}
}
// See https://manpages.debian.org/stable/apt/sources.list.5.en.html#ONE-LINE-STYLE_FORMAT
func (mgr *TemplateManager) replaceDebianMirrorUrls_OneLineStyle(handle *guestfs.Guestfs, filePath string) error {
log := mgr.logger
// Some Augeas nodes under /files/etc/apt/sources.list are comments,
// so we use /*/uri to make sure that we only get the actual entries
sourcesListEntries, err := handle.Aug_match("/files" + filePath + "/*/uri")
if err != nil {
return err
}
for _, uriPath := range sourcesListEntries {
var (
oldURL string
newURL string
typeValue string
distValue string
)
if oldURL, err = handle.Aug_get(uriPath); err != nil {
return err
}
newURL, err = mgr.replaceDebianMirrorUrl(handle, oldURL)
if err != nil {
return err
}
// strip off the "/uri" from the node path
entryPath := uriPath[:len(uriPath)-4]
typePath := entryPath + "/type"
if typeValue, err = handle.Aug_get(typePath); err != nil {
return err
}
distPath := entryPath + "/distribution"
if distValue, err = handle.Aug_get(distPath); err != nil {
return err
}
if typeValue == "deb-src" {
log.Debug().
Str("URL", oldURL).
Str("distribution", distValue).
Msg("Removing deb-src entry")
if _, err = handle.Aug_rm(entryPath); err != nil {
return err
}
continue
}
if oldURL == newURL {
continue
}
log.Debug().
Str("distribution", distValue).
Str("oldURL", oldURL).
Str("newURL", newURL).
Str("file", filePath).
Msg("Replacing URL")
if err = handle.Aug_set(uriPath, newURL); err != nil {
return err
}
}
return nil
}
// See https://manpages.debian.org/stable/apt/sources.list.5.en.html#DEB822-STYLE_FORMAT
func (mgr *TemplateManager) replaceDebianMirrorUrls_Deb822Style(handle *guestfs.Guestfs, filePath string) error {
// Augeas doesn't support this style unfortunately, so we're going to
// have to parse the file manually
content, err := handle.Cat(filePath)
if err != nil {
return fmt.Errorf("Could not read %s: %w", filePath, err)
}
oldLines := strings.Split(content, "\n")
newLines := []string{}
for _, oldLine := range oldLines {
if strings.HasPrefix(oldLine, "Types:") {
oldTypes := strings.Split(
strings.TrimLeft(oldLine[len("Types:"):], " "),
" ",
)
if len(oldTypes) == 0 || len(oldTypes) > 2 {
return fmt.Errorf(
"Could not determine Debian archive types from this line in %s: '%s'",
filePath, oldLine,
)
}
if oldTypes[0] == "deb-src" || (len(oldTypes) == 2 && oldTypes[1] == "deb-src") {
mgr.logger.Debug().Str("file", filePath).Msg("Removing deb-src type")
}
newLines = append(newLines, "Types: deb")
} else if strings.HasPrefix(oldLine, "URIs:") {
oldURL := strings.TrimLeft(oldLine[len("URIs:"):], " ")
newURL, err := mgr.replaceDebianMirrorUrl(handle, oldURL)
if err != nil {
return err
}
if oldURL != newURL {
mgr.logger.Debug().
Str("file", filePath).
Str("oldURL", oldURL).
Str("newURL", newURL).
Msg("Replacing URL")
}
newLines = append(newLines, "URIs: "+newURL)
} else {
newLines = append(newLines, oldLine)
}
}
newContent := strings.Join(newLines, "\n")
err = handle.Write(filePath, []byte(newContent))
if err != nil {
return fmt.Errorf("Error writing to %s: %w", filePath, err)
}
return nil
}
func (mgr *TemplateManager) replaceDebianMirrorUrls(handle *guestfs.Guestfs) error {
filePaths := []string{"/etc/apt/sources.list"}
extraSourceDirPath := "/etc/apt/sources.list.d"
extraSourceDirExists, err := handle.Is_dir(extraSourceDirPath, nil)
if err != nil {
return fmt.Errorf("Could not determine if %s is a directory: %w", extraSourceDirPath, err)
}
if extraSourceDirExists {
extraSourceFiles, err := handle.Ls(extraSourceDirPath)
if err != nil {
return fmt.Errorf("Could not list files in %s: %w", extraSourceDirPath, err)
}
for _, filename := range extraSourceFiles {
filePaths = append(filePaths, extraSourceDirPath+"/"+filename)
}
}
for _, filePath := range filePaths {
if strings.HasSuffix(filePath, ".list") {
err = mgr.replaceDebianMirrorUrls_OneLineStyle(handle, filePath)
} else if strings.HasSuffix(filePath, ".sources") {
err = mgr.replaceDebianMirrorUrls_Deb822Style(handle, filePath)
} else {
err = fmt.Errorf("Could not determine type of Debian source file %s", filePath)
}
if err != nil {
return err
}
}
return nil
}

65
pkg/distros/template_manager.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strings"
@ -101,10 +100,6 @@ func (mgr *TemplateManager) DownloadTemplateGeneric(filename, url string) (path
return
}
func debianCommandToUpdatePackageCache() []string {
return []string{"sudo", "apt", "update"}
}
func fedoraCommandToUpdatePackageCache() []string {
return []string{"sudo", "dnf", "makecache"}
}
@ -758,66 +753,6 @@ func (mgr *TemplateManager) replaceYumMirrorUrlsForSingleSubrepo(
return
}
// requires an Augeas handle to be open
func (mgr *TemplateManager) replaceDebianMirrorUrls(handle *guestfs.Guestfs) (err error) {
log := mgr.logger
// Some Augeas nodes under /files/etc/apt/sources.list are comments,
// so we use /*/uri to make sure that we only get the actual entries
sourcesListEntries, err := handle.Aug_match("/files/etc/apt/sources.list/*/uri")
if err != nil {
return
}
for _, uriPath := range sourcesListEntries {
var (
uriValue string
parsedUrl *url.URL
distValue string
typeValue string
)
if uriValue, err = handle.Aug_get(uriPath); err != nil {
return
}
parsedUrl, err = url.Parse(uriValue)
if err != nil {
return
}
parsedUrl.Host = mgr.cfg.MirrorHost
newUriValue := parsedUrl.String()
// strip off the "/uri" from the node path
entryPath := uriPath[:len(uriPath)-4]
typePath := entryPath + "/type"
if typeValue, err = handle.Aug_get(typePath); err != nil {
return
}
distPath := entryPath + "/distribution"
if distValue, err = handle.Aug_get(distPath); err != nil {
return
}
if typeValue == "deb-src" {
log.Debug().
Str("URL", uriValue).
Str("distribution", distValue).
Msg("Removing deb-src entry")
if _, err = handle.Aug_rm(entryPath); err != nil {
return
}
continue
}
if uriValue == newUriValue {
continue
}
log.Debug().
Str("distribution", distValue).
Str("oldURL", uriValue).
Str("newURL", newUriValue).
Msg("Replacing URL in sources.list")
if err = handle.Aug_set(uriPath, newUriValue); err != nil {
return
}
}
return
}
func (mgr *TemplateManager) dnfRemoveUnnecessaryPackages(handle *guestfs.Guestfs) (err error) {
// SSSD is unnecessary in single-user environments and consumes a lot of resources.
// auditd spams the system log and uses lots of disk IO.