From 50df5197d48b2dec6622d7bd6a4491690fb75a84 Mon Sep 17 00:00:00 2001 From: AlexSSD7 Date: Wed, 30 Aug 2023 13:13:08 +0100 Subject: [PATCH] aarch64 EFI image management --- constants/efibios.go | 27 ++++++++++++++ imgbuilder/imgbuilder.go | 7 ++-- storage/download.go | 36 +++++++++++++------ storage/storage.go | 78 ++++++++++++++++++++++++++++++---------- storage/utils.go | 30 ++++++++++++++++ vm/vm.go | 11 ++++-- 6 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 constants/efibios.go create mode 100644 storage/utils.go diff --git a/constants/efibios.go b/constants/efibios.go new file mode 100644 index 0000000..95f21ae --- /dev/null +++ b/constants/efibios.go @@ -0,0 +1,27 @@ +package constants + +import "github.com/AlexSSD7/linsk/utils" + +const aarch64EFIImageBZ2URL = "https://github.com/qemu/qemu/raw/86305e864191123dcf87c3af639fddfc59352ac6/pc-bios/edk2-aarch64-code.fd.bz2" +const aarch64EFIImageName = "edk2-aarch64-code.fd" + +var aarch64EFIImageHash []byte + +func init() { + aarch64EFIImageHash = utils.MustDecodeHex("f7f2c02853fda64cad31d4ab95ef636a7c50aac4829290e7b3a73b17d3483fc1") +} + +func GetAarch64EFIImageName() string { + return aarch64EFIImageName +} + +func GetAarch64EFIImageBZ2URL() string { + return aarch64EFIImageBZ2URL +} + +func GetAarch64EFIImageHash() []byte { + // Making a copy so that remote caller cannot modify the original variable. + tmp := make([]byte, len(aarch64EFIImageHash)) + copy(tmp, aarch64EFIImageHash) + return tmp +} diff --git a/imgbuilder/imgbuilder.go b/imgbuilder/imgbuilder.go index a28eb0f..b2890a7 100644 --- a/imgbuilder/imgbuilder.go +++ b/imgbuilder/imgbuilder.go @@ -28,7 +28,7 @@ type BuildContext struct { vi *vm.VM } -func NewBuildContext(logger *slog.Logger, baseISOPath string, outPath string, showVMDisplay bool) (*BuildContext, error) { +func NewBuildContext(logger *slog.Logger, baseISOPath string, outPath string, showVMDisplay bool, biosPath string) (*BuildContext, error) { baseISOPath = filepath.Clean(baseISOPath) outPath = filepath.Clean(outPath) @@ -50,10 +50,13 @@ func NewBuildContext(logger *slog.Logger, baseISOPath string, outPath string, sh vi, err := vm.NewVM(logger.With("subcaller", "vm"), vm.VMConfig{ CdromImagePath: baseISOPath, + BIOSPath: biosPath, Drives: []vm.DriveConfig{{ Path: outPath, }}, - MemoryAlloc: 512, + + MemoryAlloc: 512, + UnrestrictedNetworking: true, ShowDisplay: showVMDisplay, InstallBaseUtilities: true, diff --git a/storage/download.go b/storage/download.go index a80040c..a4913c1 100644 --- a/storage/download.go +++ b/storage/download.go @@ -15,7 +15,7 @@ import ( "github.com/pkg/errors" ) -func (s *Storage) download(url string, hash []byte, out string) error { +func (s *Storage) download(url string, hash []byte, out string, applyReaderMiddleware func(io.Reader) io.Reader) error { var created, success bool defer func() { @@ -51,21 +51,41 @@ func (s *Storage) download(url string, hash []byte, out string) error { defer func() { _ = resp.Body.Close() }() - _, err = copyWithProgressAndHash(f, resp.Body, 1024, resp.ContentLength, hash, func(i int, f float64) { - s.logger.Info("Downloading file", "out", out, "percent", math.Round(f*100*100)/100, "size", humanize.Bytes(uint64(resp.ContentLength))) + knownSize := resp.ContentLength + + var readFrom io.Reader + if applyReaderMiddleware != nil { + readFrom = applyReaderMiddleware(resp.Body) + knownSize = 0 + } else { + readFrom = resp.Body + } + + n, err := copyWithProgressAndHash(f, readFrom, 1024, hash, func(downloaded int) { + var percent float64 + if knownSize != 0 { + percent = float64(downloaded) / float64(knownSize) + } + + lg := s.logger.With("out", out, "done", humanize.Bytes(uint64(downloaded))) + if percent != 0 { + lg.Info("Downloading file", "percent", math.Round(percent*100*100)/100) + } else { + lg.Info("Downloading compressed file", "percent", "N/A") + } }) if err != nil { return errors.Wrap(err, "copy resp to file") } - s.logger.Info("Successfully downloaded file", "from", url, "to", out) + s.logger.Info("Successfully downloaded file", "from", url, "to", out, "out-size", humanize.Bytes(uint64(n))) success = true return nil } -func copyWithProgressAndHash(dst io.Writer, src io.Reader, blockSize int, length int64, wantHash []byte, report func(int, float64)) (int, error) { +func copyWithProgressAndHash(dst io.Writer, src io.Reader, blockSize int, wantHash []byte, report func(int)) (int, error) { block := make([]byte, blockSize) var h hash.Hash @@ -98,11 +118,7 @@ func copyWithProgressAndHash(dst io.Writer, src io.Reader, blockSize int, length } if progress%1000000 == 0 { - var percent float64 - if length != 0 { - percent = float64(progress) / float64(length) - } - report(progress, percent) + report(progress) } } diff --git a/storage/storage.go b/storage/storage.go index 5821154..c0d7c41 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,10 +1,13 @@ package storage import ( + "compress/bzip2" "fmt" + "io" "log/slog" "os" "path/filepath" + "runtime" "github.com/AlexSSD7/linsk/constants" "github.com/AlexSSD7/linsk/imgbuilder" @@ -41,7 +44,7 @@ func (s *Storage) CheckDownloadBaseImage() (string, error) { } // Image doesn't exist. Download one. - err := s.download(constants.GetAlpineBaseImageURL(), constants.GetAlpineBaseImageHash(), baseImagePath) + err := s.download(constants.GetAlpineBaseImageURL(), constants.GetAlpineBaseImageHash(), baseImagePath, nil) if err != nil { return "", errors.Wrap(err, "download base alpine image") } @@ -62,34 +65,30 @@ func (s *Storage) GetVMImagePath() string { return filepath.Join(s.path, constants.GetVMImageTags()+".qcow2") } +func (s *Storage) GetAarch64EFIImagePath() string { + return filepath.Join(s.path, constants.GetAarch64EFIImageName()) +} + func (s *Storage) BuildVMImageWithInterruptHandler(showBuilderVMDisplay bool, overwrite bool) error { - var overwriting bool vmImagePath := s.GetVMImagePath() - _, err := os.Stat(vmImagePath) + removed, err := checkExistsOrRemove(vmImagePath, overwrite) if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return errors.Wrap(err, "stat vm image path") - } - } else { - if overwrite { - overwriting = true - err = os.Remove(vmImagePath) - if err != nil { - return errors.Wrap(err, "remove existing vm image") - } - } else { - return ErrImageAlreadyExists - } + return errors.Wrap(err, "check exists or remove") } baseImagePath, err := s.CheckDownloadBaseImage() if err != nil { - return errors.Wrap(err, "check download base image") + return errors.Wrap(err, "check/download base image") } - s.logger.Info("Building VM image", "tags", constants.GetAlpineBaseImageTags(), "overwriting", overwriting, "dst", vmImagePath) + biosPath, err := s.CheckDownloadCPUArchSpecifics() + if err != nil { + return errors.Wrap(err, "check/download cpu arch specifics") + } - buildCtx, err := imgbuilder.NewBuildContext(s.logger.With("subcaller", "imgbuilder"), baseImagePath, vmImagePath, showBuilderVMDisplay) + s.logger.Info("Building VM image", "tags", constants.GetAlpineBaseImageTags(), "overwriting", removed, "dst", vmImagePath) + + buildCtx, err := imgbuilder.NewBuildContext(s.logger.With("subcaller", "imgbuilder"), baseImagePath, vmImagePath, showBuilderVMDisplay, biosPath) if err != nil { return errors.Wrap(err, "create new img build context") } @@ -116,3 +115,44 @@ func (s *Storage) CheckVMImageExists() (string, error) { func (s *Storage) DataDirPath() string { return s.path } + +func (s *Storage) CheckDownloadCPUArchSpecifics() (string, error) { + if runtime.GOARCH == "arm64" { + p, err := s.CheckDownloadAarch64EFIImage() + if err != nil { + return "", errors.Wrap(err, "check/download aarch64 efi image") + } + + return p, nil + } + + return "", nil +} + +func (s *Storage) CheckDownloadAarch64EFIImage() (string, error) { + efiImagePath := s.GetAarch64EFIImagePath() + _, err := os.Stat(efiImagePath) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return "", errors.Wrap(err, "stat base image path") + } + + // EFI image doesn't exist. Download one. + err := s.download(constants.GetAarch64EFIImageBZ2URL(), constants.GetAarch64EFIImageHash(), efiImagePath, func(r io.Reader) io.Reader { + return bzip2.NewReader(r) + }) + if err != nil { + return "", errors.Wrap(err, "download base alpine image") + } + + return efiImagePath, nil + } + + // EFI image exists. Ensure that the hash is correct. + err = validateFileHash(efiImagePath, constants.GetAarch64EFIImageHash()) + if err != nil { + return "", errors.Wrap(err, "validate hash of existing image") + } + + return efiImagePath, nil +} diff --git a/storage/utils.go b/storage/utils.go new file mode 100644 index 0000000..3eaec96 --- /dev/null +++ b/storage/utils.go @@ -0,0 +1,30 @@ +package storage + +import ( + "os" + + "github.com/pkg/errors" +) + +func checkExistsOrRemove(path string, overwriteRemove bool) (bool, error) { + var removed bool + + _, err := os.Stat(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return removed, errors.Wrap(err, "stat file") + } + } else { + if overwriteRemove { + err = os.Remove(path) + if err != nil { + return removed, errors.Wrap(err, "remove file") + } + removed = true + } else { + return removed, ErrImageAlreadyExists + } + } + + return removed, nil +} diff --git a/vm/vm.go b/vm/vm.go index 5e15de6..67714d0 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -65,6 +65,7 @@ type DriveConfig struct { type VMConfig struct { CdromImagePath string + BIOSPath string Drives []DriveConfig MemoryAlloc uint32 // In KiB. @@ -96,6 +97,10 @@ func NewVM(logger *slog.Logger, cfg VMConfig) (*VM, error) { cmdArgs := []string{"-serial", "stdio", "-m", fmt.Sprint(cfg.MemoryAlloc), "-smp", fmt.Sprint(runtime.NumCPU())} + if cfg.BIOSPath != "" { + cmdArgs = append(cmdArgs, "-bios", filepath.Clean(cfg.BIOSPath)) + } + baseCmd := "qemu-system" switch runtime.GOARCH { @@ -112,8 +117,10 @@ func NewVM(logger *slog.Logger, cfg VMConfig) (*VM, error) { cmdArgs = append(cmdArgs, "-accel", accel) baseCmd += "-x86_64" case "arm64": - // TODO: EFI firmware path is temporary, for dev purposes only. - cmdArgs = append(cmdArgs, "-accel", "hvf", "-bios", "/opt/homebrew/Cellar/qemu/8.1.0/share/qemu/edk2-aarch64-code.fd", "-M", "virt,highmem=off", "-cpu", "cortex-a57") + if cfg.BIOSPath == "" { + logger.Warn("BIOS image path is not specified while attempting to run an aarch64 (arm64) VM. The VM will not boot.") + } + cmdArgs = append(cmdArgs, "-accel", "hvf", "-M", "virt,highmem=off", "-cpu", "cortex-a57") baseCmd += "-aarch64" default: return nil, fmt.Errorf("arch '%v' is not supported", runtime.GOARCH)