Local VM image building

storage/download.go

@@ -0,0 +1,117 @@
+package storage
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"hash"
+	"io"
+	"math"
+	"net/http"
+	"os"
+
+	"github.com/dustin/go-humanize"
+	"github.com/pkg/errors"
+)
+
+func (s *Storage) download(url string, hash []byte, out string) error {
+	var created, success bool
+
+	defer func() {
+		if created && !success {
+			_ = os.Remove(out)
+		}
+	}()
+
+	_, err := os.Stat(out)
+	if err == nil {
+		return errors.Wrap(err, "file already exists")
+	} else {
+		if !errors.Is(err, os.ErrNotExist) {
+			return errors.Wrap(err, "stat out path")
+		}
+	}
+
+	f, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY, 0400)
+	if err != nil {
+		return errors.Wrap(err, "open file")
+	}
+
+	created = true
+
+	defer func() { _ = f.Close() }()
+
+	s.logger.Info("Starting to download file", "from", url, "to", out)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return errors.Wrap(err, "http get")
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	_, err = copyWithProgressAndHash(f, resp.Body, 1024, resp.ContentLength, hash, func(i int, f float64) {
+		s.logger.Info("Downloading file", "out", out, "percent", math.Round(f*100*100)/100, "size", humanize.Bytes(uint64(resp.ContentLength)))
+	})
+	if err != nil {
+		return errors.Wrap(err, "copy resp to file")
+	}
+
+	s.logger.Info("Successfully downloaded file", "from", url, "to", out)
+
+	success = true
+
+	return nil
+}
+
+func copyWithProgressAndHash(dst io.Writer, src io.Reader, blockSize int, length int64, wantHash []byte, report func(int, float64)) (int, error) {
+	block := make([]byte, blockSize)
+
+	var h hash.Hash
+	if wantHash != nil {
+		h = sha256.New()
+	}
+
+	var progress int
+
+	for {
+		read, err := src.Read(block)
+		if read > 0 {
+			written, err := dst.Write(block[:read])
+			if err != nil {
+				return progress, errors.Wrap(err, "write")
+			}
+
+			if h != nil {
+				h.Write(block[:read])
+			}
+
+			progress += written
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			return progress, errors.Wrap(err, "read")
+		}
+
+		if progress%1000000 == 0 {
+			var percent float64
+			if length != 0 {
+				percent = float64(progress) / float64(length)
+			}
+			report(progress, percent)
+		}
+	}
+
+	if h != nil {
+		sum := h.Sum(nil)
+		if !bytes.Equal(sum, wantHash) {
+			return progress, fmt.Errorf("hash mismach: want '%v', have '%v'", hex.EncodeToString(wantHash), hex.EncodeToString(sum))
+		}
+	}
+
+	return progress, nil
+}