Local VM image building

cmd/build.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"log/slog"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var buildCmd = &cobra.Command{
+	Use:   "build",
+	Short: "Build (set up) a VM image for local use. This needs to be run after the initial installation.",
+	Run: func(cmd *cobra.Command, args []string) {
+		store := createStore()
+
+		err := store.BuildVMImageWithInterruptHandler(vmDebugFlag, buildOverwriteFlag)
+		if err != nil {
+			slog.Error("Failed to build VM image", "error", err.Error())
+			os.Exit(1)
+		}
+
+		slog.Info("VM image built successfully", "path", store.GetVMImagePath())
+	},
+}
+
+var buildOverwriteFlag bool
+
+func init() {
+	buildCmd.Flags().BoolVar(&buildOverwriteFlag, "overwrite", false, "Specifies whether the VM image should be overwritten with the build.")
+}

cmd/clean.go

@@ -12,11 +12,12 @@ import (
 
 var cleanCmd = &cobra.Command{
 	Use:   "clean",
-	Short: "Remove all downloaded VM images.",
+	Short: "Remove the Linsk data directory.",
 	Run: func(cmd *cobra.Command, args []string) {
 		store := createStore()
 
-		fmt.Fprintf(os.Stderr, "Will delete all VM images in the data directory. Proceed? (y/n) > ")
+		rmPath := store.DataDirPath()
+		fmt.Fprintf(os.Stderr, "Will permanently remove '"+rmPath+"'. Proceed? (y/n) > ")
 
 		reader := bufio.NewReader(os.Stdin)
 		answer, err := reader.ReadBytes('\n')
@@ -30,12 +31,12 @@ var cleanCmd = &cobra.Command{
 			os.Exit(2)
 		}
 
-		deleted, err := store.CleanImages(false)
+		err = os.RemoveAll(rmPath)
 		if err != nil {
-			slog.Error("Failed to clean images", "error", err.Error())
+			slog.Error("Failed to remove all", "error", err.Error(), "path", rmPath)
 			os.Exit(1)
 		}
 
-		slog.Info("Successful VM image cleanup", "deleted", deleted)
+		slog.Info("Deleted data directory", "path", rmPath)
 	},
 }

cmd/imgbuilder/main.go

@@ -1,49 +0,0 @@
-package main
-
-import (
-	"log/slog"
-	"os"
-	"path/filepath"
-
-	"github.com/AlexSSD7/linsk/cmd/imgbuilder/builder"
-	"github.com/spf13/cobra"
-)
-
-var rootCmd = &cobra.Command{
-	Use:   "imgbuilder",
-	Short: "Build an Alpine Linux image for Linsk. A base Alpine VM disc image is required.",
-	Args:  cobra.ExactArgs(2),
-	Run: func(cmd *cobra.Command, args []string) {
-		baseISOPath := filepath.Clean(args[0])
-		outImagePath := filepath.Clean(args[1])
-
-		bc, err := builder.NewBuildContext(slog.With("caller", "build-context"), baseISOPath, outImagePath, vmDebugFlag)
-		if err != nil {
-			slog.Error("Failed to create a new build context", "error", err.Error())
-			os.Exit(1)
-		}
-
-		err = bc.BuildWithInterruptHandler()
-		if err != nil {
-			slog.Error("Failed to build an image", "error", err.Error())
-			os.Exit(1)
-		}
-
-		slog.Info("Success")
-	},
-}
-
-var vmDebugFlag bool
-
-func init() {
-	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
-
-	rootCmd.PersistentFlags().BoolVar(&vmDebugFlag, "vmdebug", false, "Enable VM debug mode. This will open an accessible VM monitor. You can log in with root user and no password.")
-}
-
-func main() {
-	err := rootCmd.Execute()
-	if err != nil {
-		os.Exit(1)
-	}
-}

cmd/root.go

@@ -42,6 +42,7 @@ func init() {
 	rootCmd.AddCommand(runCmd)
 	rootCmd.AddCommand(shellCmd)
 	rootCmd.AddCommand(cleanCmd)
+	rootCmd.AddCommand(buildCmd)
 
 	rootCmd.PersistentFlags().BoolVar(&vmDebugFlag, "vmdebug", false, "Enables the VM debug mode. This will open an accessible VM monitor. You can log in with root user and no password.")
 	rootCmd.PersistentFlags().BoolVar(&unrestrictedNetworkingFlag, "unrestricted-networking", false, "Enables unrestricted networking. This will allow the VM to connect to the internet.")

cmd/utils.go

@@ -66,9 +66,15 @@ func createStore() *storage.Storage {
 
 func runVM(passthroughArg string, fn func(context.Context, *vm.VM, *vm.FileManager) int, forwardPortsRules []vm.PortForwardingRule, unrestrictedNetworking bool) int {
 	store := createStore()
-	_, err := store.ValidateImageHashOrDownload()
+
+	vmImagePath, err := store.CheckVMImageExists()
 	if err != nil {
-		slog.Error("Failed to validate image hash or download image", "error", err.Error())
+		slog.Error("Failed to check whether VM image exists", "error", err.Error())
+		os.Exit(1)
+	}
+
+	if vmImagePath == "" {
+		slog.Error("VM image does not exist. You need to build it first before attempting to start Linsk. Please run `linsk build` first.")
 		os.Exit(1)
 	}
 
@@ -81,7 +87,7 @@ func runVM(passthroughArg string, fn func(context.Context, *vm.VM, *vm.FileManag
 
 	vmCfg := vm.VMConfig{
 		Drives: []vm.DriveConfig{{
-			Path:         store.GetLocalImagePath(),
+			Path:         vmImagePath,
 			SnapshotMode: true,
 		}},
 

constants/arch.go

@@ -0,0 +1,14 @@
+package constants
+
+import "runtime"
+
+func GetUnixWorkArch() string {
+	arch := "x86_64"
+	if runtime.GOOS == "arm64" {
+		arch = "arm64"
+	}
+
+	// CPU architectures other than amd64 and arm64 are not yet natively supported.
+	// Running on a non-officially-supported arch will result in use of x86_64 VM.
+	return arch
+}

constants/image.go

@@ -0,0 +1,51 @@
+package constants
+
+import (
+	"runtime"
+
+	"github.com/AlexSSD7/linsk/utils"
+)
+
+const baseAlpineVersionMajor = "3.18"
+const baseAlpineVersionMinor = "3"
+const baseAlpineVersionCombined = baseAlpineVersionMajor + "." + baseAlpineVersionMinor
+
+const LinskVMImageVersion = "1"
+
+var baseAlpineArch string
+var baseImageURL string
+var alpineBaseImageHash []byte
+
+func init() {
+	baseAlpineArch = "x86_64"
+	alpineBaseImageHash = utils.MustDecodeHex("925f6bc1039a0abcd0548d2c3054d54dce31cfa03c7eeba22d10d85dc5817c98")
+	if runtime.GOOS == "arm64" {
+		baseAlpineArch = "aarch64"
+		alpineBaseImageHash = utils.MustDecodeHex("c94593729e4577650d9e73ada28e3cbe56964ab2a27240364f8616e920ed6d4e")
+	}
+
+	baseImageURL = "https://dl-cdn.alpinelinux.org/alpine/v" + baseAlpineVersionMajor + "/releases/" + baseAlpineArch + "/alpine-virt-" + baseAlpineVersionCombined + "-" + baseAlpineArch + ".iso"
+}
+
+func GetAlpineBaseImageURL() string {
+	return baseImageURL
+}
+
+func GetAlpineBaseImageTags() string {
+	return baseAlpineVersionCombined + "-" + baseAlpineArch
+}
+
+func GetVMImageTags() string {
+	return GetAlpineBaseImageTags() + "-linsk" + LinskVMImageVersion
+}
+
+func GetAlpineBaseImageFileName() string {
+	return "alpine-" + GetAlpineBaseImageTags() + ".img"
+}
+
+func GetAlpineBaseImageHash() []byte {
+	// Making a copy so that remote caller cannot modify the original variable.
+	tmp := make([]byte, len(alpineBaseImageHash))
+	copy(tmp, alpineBaseImageHash)
+	return tmp
+}

imgbuilder/imgbuilder.go

@@ -1,4 +1,4 @@
-package builder
+package imgbuilder
 
 import (
 	"bytes"
@@ -152,7 +152,7 @@ func (bc *BuildContext) BuildWithInterruptHandler() error {
 
 			defer func() { _ = sc.Close() }()
 
-			bc.logger.Info("Installation in progress")
+			bc.logger.Info("VM OS installation in progress")
 
 			err = runAlpineSetupCmd(sc, []string{"openssh", "lvm2", "util-linux", "cryptsetup", "vsftpd"})
 			if err != nil {

storage/download.go

@@ -0,0 +1,117 @@
+package storage
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"io"
+	"math"
+	"net/http"
+	"os"
+
+	"github.com/dustin/go-humanize"
+	"github.com/pkg/errors"
+)
+
+func (s *Storage) download(url string, hash []byte, out string) error {
+	var created, success bool
+
+	defer func() {
+		if created && !success {
+			_ = os.Remove(out)
+		}
+	}()
+
+	_, err := os.Stat(out)
+	if err == nil {
+		return errors.Wrap(err, "file already exists")
+	} else {
+		if !errors.Is(err, os.ErrNotExist) {
+			return errors.Wrap(err, "stat out path")
+		}
+	}
+
+	f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY, 0400)
+	if err != nil {
+		return errors.Wrap(err, "open file")
+	}
+
+	created = true
+
+	defer func() { _ = f.Close() }()
+
+	s.logger.Info("Starting to download file", "from", url, "to", out)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return errors.Wrap(err, "http get")
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	_, err = copyWithProgressAndHash(f, resp.Body, 1024, resp.ContentLength, hash, func(i int, f float64) {
+		s.logger.Info("Downloading file", "out", out, "percent", math.Round(f*100*100)/100, "size", humanize.Bytes(uint64(resp.ContentLength)))
+	})
+	if err != nil {
+		return errors.Wrap(err, "copy resp to file")
+	}
+
+	s.logger.Info("Successfully downloaded file", "from", url, "to", out)
+
+	success = true
+
+	return nil
+}
+
+func copyWithProgressAndHash(dst io.Writer, src io.Reader, blockSize int, length int64, wantHash []byte, report func(int, float64)) (int, error) {
+	block := make([]byte, blockSize)
+
+	var h hash.Hash
+	if wantHash != nil {
+		h = sha256.New()
+	}
+
+	var progress int
+
+	for {
+		read, err := src.Read(block)
+		if read > 0 {
+			written, err := dst.Write(block[:read])
+			if err != nil {
+				return progress, errors.Wrap(err, "write")
+			}
+
+			if h != nil {
+				h.Write(block[:read])
+			}
+
+			progress += written
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			return progress, errors.Wrap(err, "read")
+		}
+
+		if progress%1000000 == 0 {
+			var percent float64
+			if length != 0 {
+				percent = float64(progress) / float64(length)
+			}
+			report(progress, percent)
+		}
+	}
+
+	if h != nil {
+		sum := h.Sum(nil)
+		if !bytes.Equal(sum, wantHash) {
+			return progress, fmt.Errorf("hash mismach: want '%v', have '%v'", hex.EncodeToString(wantHash), hex.EncodeToString(sum))
+		}
+	}
+
+	return progress, nil
+}

storage/errors.go

@@ -0,0 +1,7 @@
+package storage
+
+import "errors"
+
+var (
+	ErrImageAlreadyExists = errors.New("image already exists")
+)

storage/hash.go

@@ -0,0 +1,45 @@
+package storage
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+func validateFileHash(path string, hash []byte) error {
+	f, err := os.OpenFile(path, os.O_RDONLY, 0400)
+	if err != nil {
+		return errors.Wrap(err, "open file")
+	}
+
+	defer func() { _ = f.Close() }()
+
+	h := sha256.New()
+	block := make([]byte, 1024)
+	for {
+		read, err := f.Read(block)
+		if read > 0 {
+			h.Write(block[:read])
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			return errors.Wrap(err, "read file block")
+		}
+	}
+
+	sum := h.Sum(nil)
+
+	if !bytes.Equal(sum, hash) {
+		return fmt.Errorf("hash mismatch: want '%v', have '%v'", hex.EncodeToString(hash), hex.EncodeToString(sum))
+	}
+
+	return nil
+}

storage/storage.go

@@ -1,26 +1,16 @@
 package storage
 
 import (
-	"bytes"
-	"crypto/sha512"
-	"encoding/hex"
 	"fmt"
-	"io"
 	"log/slog"
-	"math"
-	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 
-	"github.com/dustin/go-humanize"
+	"github.com/AlexSSD7/linsk/constants"
+	"github.com/AlexSSD7/linsk/imgbuilder"
 	"github.com/pkg/errors"
 )
 
-const imageURL = "http://localhost:8000/linsk-base.qcow2"
-
-var imageHash = [64]byte{70, 23, 243, 131, 146, 197, 41, 223, 67, 223, 41, 243, 128, 147, 82, 238, 34, 24, 123, 246, 251, 117, 120, 72, 72, 64, 96, 146, 227, 199, 49, 169, 164, 33, 205, 217, 98, 255, 109, 18, 130, 203, 126, 83, 34, 4, 229, 108, 173, 22, 107, 37, 181, 17, 84, 13, 129, 110, 25, 126, 158, 50, 135, 9}
-
 type Storage struct {
 	logger *slog.Logger
 
@@ -42,192 +32,87 @@ func NewStorage(logger *slog.Logger, dataDir string) (*Storage, error) {
 	}, nil
 }
 
-func (s *Storage) CleanImages(retainLatest bool) (int, error) {
+func (s *Storage) CheckDownloadBaseImage() (string, error) {
-	dirEntries, err := os.ReadDir(s.path)
+	baseImagePath := filepath.Join(s.path, constants.GetAlpineBaseImageFileName())
+	_, err := os.Stat(baseImagePath)
 	if err != nil {
-		return 0, errors.Wrap(err, "read dir")
-	}
-
-	localImagePath := s.GetLocalImagePath()
-
-	var deleted int
-
-	for _, entry := range dirEntries {
-		fullPath := filepath.Join(s.path, entry.Name())
-		if !strings.HasSuffix(fullPath, ".qcow2") {
-			continue
-		}
-
-		if !(retainLatest && fullPath == localImagePath) {
-			s.logger.Warn("Removing VM image", "path", fullPath)
-			err = os.Remove(fullPath)
-			if err != nil {
-				return deleted, errors.Wrapf(err, "remove vm image (path '%v')", fullPath)
-			}
-			deleted++
-		}
-	}
-
-	return deleted, nil
-}
-
-func (s *Storage) GetLocalImagePath() string {
-	return filepath.Join(s.path, hex.EncodeToString(imageHash[:])+".qcow2")
-}
-
-func (s *Storage) DownloadImage() error {
-	localImagePath := s.GetLocalImagePath()
-
-	var created, success bool
-
-	defer func() {
-		if created && !success {
-			_ = os.Remove(localImagePath)
-		}
-	}()
-
-	_, err := os.Stat(localImagePath)
-	if err == nil {
-		err = os.Remove(localImagePath)
-		if err != nil {
-			return errors.Wrap(err, "remove existing image")
-		}
-	} else {
 		if !errors.Is(err, os.ErrNotExist) {
-			return errors.Wrap(err, "stat local image path")
+			return "", errors.Wrap(err, "stat base image path")
 		}
-	}
 
-	f, err := os.OpenFile(localImagePath, os.O_CREATE|os.O_WRONLY, 0400)
+		// Image doesn't exist. Download one.
-	if err != nil {
+		err := s.download(constants.GetAlpineBaseImageURL(), constants.GetAlpineBaseImageHash(), baseImagePath)
-		return errors.Wrap(err, "open file")
-	}
-
-	created = true
-
-	defer func() { _ = f.Close() }()
-
-	s.logger.Info("Starting to download the VM image", "path", localImagePath)
-
-	resp, err := http.Get(imageURL)
-	if err != nil {
-		return errors.Wrap(err, "http get image")
-	}
-
-	defer func() { _ = resp.Body.Close() }()
-
-	_, err = copyWithProgress(f, resp.Body, 1024, resp.ContentLength, func(i int, f float64) {
-		s.logger.Info("Downloading the VM image", "url", imageURL, "percent", math.Round(f*100*100)/100, "content-length", humanize.Bytes(uint64(resp.ContentLength)))
-	})
-	if err != nil {
-		return errors.Wrap(err, "copy resp to file")
-	}
-
-	err = s.ValidateImageHash()
-	if err != nil {
-		return errors.Wrap(err, "validate image hash")
-	}
-
-	s.logger.Info("Successfully downloaded the VM image", "dst", localImagePath)
-
-	success = true
-
-	return nil
-}
-
-func (s *Storage) ValidateImageHash() error {
-	localImagePath := s.GetLocalImagePath()
-
-	f, err := os.OpenFile(localImagePath, os.O_RDONLY, 0400)
-	if err != nil {
-		return errors.Wrap(err, "open file")
-	}
-
-	defer func() { _ = f.Close() }()
-
-	h := sha512.New()
-	block := make([]byte, 1024)
-	for {
-		read, err := f.Read(block)
-		if read > 0 {
-			h.Write(block[:read])
-		}
 		if err != nil {
-			if errors.Is(err, io.EOF) {
+			return "", errors.Wrap(err, "download base alpine image")
-				break
-			}
-
-			return errors.Wrap(err, "read file block")
 		}
+
+		return baseImagePath, nil
 	}
 
-	sum := h.Sum(nil)
+	// Image exists. Ensure that the hash is correct.
-
+	err = validateFileHash(baseImagePath, constants.GetAlpineBaseImageHash())
-	if !bytes.Equal(sum, imageHash[:]) {
+	if err != nil {
-		return fmt.Errorf("hash mismatch: want '%v', have '%v'", hex.EncodeToString(imageHash[:]), hex.EncodeToString(sum))
+		return "", errors.Wrap(err, "validate hash of existing image")
 	}
 
-	s.logger.Info("Validated the VM image hash", "path", localImagePath)
+	return baseImagePath, nil
-
-	return nil
 }
 
-func (s *Storage) ValidateImageHashOrDownload() (bool, error) {
+func (s *Storage) GetVMImagePath() string {
-	var downloaded bool
+	return filepath.Join(s.path, constants.GetVMImageTags()+".qcow2")
-	err := s.ValidateImageHash()
+}
-	if err != nil {
-		if errors.Is(err, os.ErrNotExist) {
-			err = s.DownloadImage()
-			if err != nil {
-				return false, errors.Wrap(err, "download iamge")
-			}
 
-			downloaded = true
+func (s *Storage) BuildVMImageWithInterruptHandler(showBuilderVMDisplay bool, overwrite bool) error {
-		} else {
+	var overwriting bool
-			return false, errors.Wrap(err, "validate image hash")
+	vmImagePath := s.GetVMImagePath()
+	_, err := os.Stat(vmImagePath)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return errors.Wrap(err, "stat vm image path")
 		}
-	}
-
-	deletedImagesCount, err := s.CleanImages(true)
-	if err != nil {
-		s.logger.Warn("Failed to prune old VM images", "error", err, "deleted", deletedImagesCount)
 	} else {
-		s.logger.Info("Pruned old VM images", "deleted", deletedImagesCount)
+		if overwrite {
-	}
+			overwriting = true
-
+			err = os.Remove(vmImagePath)
-	return downloaded, err
-}
-
-func copyWithProgress(dst io.Writer, src io.Reader, blockSize int, length int64, report func(int, float64)) (int, error) {
-	block := make([]byte, blockSize)
-
-	var progress int
-
-	for {
-		read, err := src.Read(block)
-		if read > 0 {
-			written, err := dst.Write(block[:read])
 			if err != nil {
-				return progress, errors.Wrap(err, "write")
+				return errors.Wrap(err, "remove existing vm image")
 			}
-			progress += written
+		} else {
-		}
+			return ErrImageAlreadyExists
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-
-			return progress, errors.Wrap(err, "read")
-		}
-
-		if progress%1000000 == 0 {
-			var percent float64
-			if length != 0 {
-				percent = float64(progress) / float64(length)
-			}
-			report(progress, percent)
 		}
 	}
 
-	return progress, nil
+	baseImagePath, err := s.CheckDownloadBaseImage()
+	if err != nil {
+		return errors.Wrap(err, "check download base image")
+	}
+
+	s.logger.Info("Building VM image", "tags", constants.GetAlpineBaseImageTags(), "overwriting", overwriting, "dst", vmImagePath)
+
+	buildCtx, err := imgbuilder.NewBuildContext(s.logger.With("subcaller", "imgbuilder"), baseImagePath, vmImagePath, showBuilderVMDisplay)
+	if err != nil {
+		return errors.Wrap(err, "create new img build context")
+	}
+
+	return errors.Wrap(buildCtx.BuildWithInterruptHandler(), "build")
+}
+
+func (s *Storage) CheckVMImageExists() (string, error) {
+	p := s.GetVMImagePath()
+	_, err := os.Stat(p)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return "", errors.Wrap(err, "stat vm image path")
+		}
+
+		// Image doesn't exist.
+		return "", nil
+	}
+
+	// Image exists. Returning the full path.
+	return p, nil
+}
+
+func (s *Storage) DataDirPath() string {
+	return s.path
 }

utils/hex.go

@@ -0,0 +1,12 @@
+package utils
+
+import "encoding/hex"
+
+func MustDecodeHex(s string) []byte {
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return b
+}