feat: add sha256sum check (#832)

* feat(download): add sha256sum check and cancellable downloads

+ server/proto/download.go
	+ add DownloadImageReq and sha256 argument

server/router/download.go
	+ add cancel download router

server/service/download/service.go
	+ sha256 decode and check
	+ remove HEAD request
	+ cancel download

* refactor(download): synchronize image and application transfers

sovel the the problem that image downloading and app downloading would conflict
at same time

* fix: optimized UI of image download and locale issue

* fix: locale
This commit is contained in:
Guoguo
2026-07-17 18:17:04 +08:00
committed by GitHub
31 changed files with 872 additions and 382 deletions

View File

@@ -1,5 +1,10 @@
package proto
type DownloadImageReq struct {
File string `json:"file" validate:"required"`
SHA256Sum string `json:"sha256sum"`
}
type ImageEnabledRsp struct {
Enabled bool `json:"enabled"`
}

View File

@@ -11,8 +11,9 @@ func downloadRouter(r *gin.Engine) {
service := download.NewService()
api := r.Group("/api").Use(middleware.CheckToken())
api.POST("/download/image", service.DownloadImage) // download image
api.GET("/download/image/status", service.StatusImage) // download image
api.GET("/download/image/enabled", service.ImageEnabled) // download image
api.POST("/download/file", service.DownloadImageFile) // download image
api.POST("/download/image", service.DownloadImage) // download image
api.POST("/download/image/cancel", service.CancelDownloadImage) // cancel image download
api.GET("/download/image/status", service.StatusImage) // download image
api.GET("/download/image/enabled", service.ImageEnabled) // download image
api.POST("/download/file", service.DownloadImageFile) // download image
}

View File

@@ -46,10 +46,6 @@ func offlineUpdate(c *gin.Context) error {
_ = os.RemoveAll(CacheDir)
}()
if err := checkDownloadInProgress(); err != nil {
return err
}
if err := createSentinelFile(); err != nil {
return err
}
@@ -75,19 +71,30 @@ func offlineUpdate(c *gin.Context) error {
return nil
}
func checkDownloadInProgress() error {
if _, err := os.Stat(sentinelPath); err == nil {
log.Debug("Download in progress")
return fmt.Errorf("download already in progress")
}
return nil
}
func createSentinelFile() error {
if err := os.WriteFile(sentinelPath, []byte("downloading"), sentinelPermission); err != nil {
file, err := os.OpenFile(
sentinelPath,
os.O_WRONLY|os.O_CREATE|os.O_EXCL,
sentinelPermission,
)
if err != nil {
if os.IsExist(err) {
return fmt.Errorf("download already in progress")
}
log.Errorf("Failed to create sentinel file: %v", err)
return fmt.Errorf("failed to create sentinel file: %w", err)
}
if _, err := file.WriteString("downloading"); err != nil {
_ = file.Close()
_ = os.Remove(sentinelPath)
return fmt.Errorf("failed to initialize sentinel file: %w", err)
}
if err := file.Close(); err != nil {
_ = os.Remove(sentinelPath)
return fmt.Errorf("failed to close sentinel file: %w", err)
}
return nil
}

View File

@@ -2,54 +2,165 @@ package download
import (
"NanoKVM-Server/proto"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
type Service struct{}
type downloadStatus string
var sentinelPath = "/tmp/.download_in_progress"
const (
transferSentinelPath = "/tmp/.download_in_progress"
downloadStatusIdle downloadStatus = "idle"
downloadStatusInProgress downloadStatus = "in_progress"
downloadStatusSuccess downloadStatus = "success"
downloadStatusFailed downloadStatus = "failed"
downloadStatusChecksumFailed downloadStatus = "checksum_failed"
)
var (
errDownloadInProgress = errors.New("download in progress")
errSHA256Mismatch = errors.New("sha256 mismatch")
validISOFilename = regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
)
type Service struct {
downloadMutex sync.Mutex
downloadCancel context.CancelFunc
downloadDone chan struct{}
downloadStatus downloadStatus
downloadFile string
downloadPercentage string
}
func NewService() *Service {
// Clear sentinel
// If we are starting from scratch, we need to remove the sentinel file as any downloads at this point are done or broken
_ = os.Remove(sentinelPath)
return &Service{}
// A running transfer cannot survive a server restart, so any remaining lock
// file is stale at this point.
_ = os.Remove(transferSentinelPath)
return &Service{downloadStatus: downloadStatusIdle}
}
func (s *Service) CancelDownloadImage(c *gin.Context) {
var rsp proto.Response
s.downloadMutex.Lock()
cancel := s.downloadCancel
done := s.downloadDone
s.downloadMutex.Unlock()
if cancel == nil || done == nil {
rsp.ErrRsp(c, -1, "no cancellable download in progress")
return
}
cancel()
select {
case <-done:
rsp.OkRsp(c)
case <-time.After(10 * time.Second):
rsp.ErrRsp(c, -1, "cancel download timed out")
}
}
func (s *Service) beginDownload(file string, cancel context.CancelFunc) (chan struct{}, error) {
s.downloadMutex.Lock()
defer s.downloadMutex.Unlock()
if s.downloadDone != nil {
return nil, errDownloadInProgress
}
lockFile, err := os.OpenFile(
transferSentinelPath,
os.O_WRONLY|os.O_CREATE|os.O_EXCL,
0o644,
)
if err != nil {
if os.IsExist(err) {
return nil, errDownloadInProgress
}
return nil, fmt.Errorf("acquire transfer lock failed: %w", err)
}
if err := lockFile.Close(); err != nil {
_ = os.Remove(transferSentinelPath)
return nil, fmt.Errorf("close transfer lock failed: %w", err)
}
done := make(chan struct{})
s.downloadCancel = cancel
s.downloadDone = done
s.downloadStatus = downloadStatusInProgress
s.downloadFile = file
s.downloadPercentage = ""
return done, nil
}
func (s *Service) setDownloadFile(done chan struct{}, file string) {
s.downloadMutex.Lock()
defer s.downloadMutex.Unlock()
if s.downloadDone == done {
s.downloadFile = file
}
}
func (s *Service) setDownloadProgress(done chan struct{}, percentage string) {
s.downloadMutex.Lock()
defer s.downloadMutex.Unlock()
if s.downloadDone == done {
s.downloadPercentage = percentage
}
}
func (s *Service) finishDownload(done chan struct{}, status downloadStatus) {
s.downloadMutex.Lock()
if s.downloadDone != done {
s.downloadMutex.Unlock()
return
}
s.downloadCancel = nil
s.downloadDone = nil
s.downloadStatus = status
s.downloadFile = ""
s.downloadPercentage = ""
_ = os.Remove(transferSentinelPath)
s.downloadMutex.Unlock()
close(done)
}
func (s *Service) ImageEnabled(c *gin.Context) {
var rsp proto.Response
// Check if /data mount is RO/RW
testFile := "/data/.testfile"
file, err := os.Create(testFile)
defer file.Close()
defer os.Remove(testFile)
if err != nil {
if os.IsPermission(err) {
rsp.OkRspWithData(c, &proto.ImageEnabledRsp{
Enabled: false,
})
return
}
rsp.OkRspWithData(c, &proto.ImageEnabledRsp{
Enabled: false,
})
rsp.OkRspWithData(c, &proto.ImageEnabledRsp{Enabled: false})
return
}
defer file.Close()
defer os.Remove(testFile)
rsp.OkRspWithData(c, &proto.ImageEnabledRsp{
Enabled: true,
})
rsp.OkRspWithData(c, &proto.ImageEnabledRsp{Enabled: true})
}
func isISO9660(path string) (bool, error) {
@@ -59,15 +170,13 @@ func isISO9660(path string) (bool, error) {
}
defer f.Close()
// ISO-9660 Magic "CD001" bei Offset 32769
_, err = f.Seek(0x8001, io.SeekStart)
if err != nil {
// ISO-9660 magic "CD001" at offset 32769.
if _, err = f.Seek(0x8001, io.SeekStart); err != nil {
return false, err
}
buf := make([]byte, 5)
_, err = io.ReadFull(f, buf)
if err != nil {
if _, err = io.ReadFull(f, buf); err != nil {
return false, err
}
@@ -77,211 +186,141 @@ func isISO9660(path string) (bool, error) {
func (s *Service) StatusImage(c *gin.Context) {
var rsp proto.Response
// Check if the sentinel file exists
log.Debug("StatusImage")
if _, err := os.Stat(sentinelPath); err == nil {
content, err := os.ReadFile(sentinelPath)
if err != nil {
log.Error("Failed to read sentinel file")
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: "in_progress",
File: "",
Percentage: "",
})
return
}
splitted := strings.Split(string(content), ";")
if len(splitted) == 1 {
// No percentage, just the URL
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: "in_progress",
File: splitted[0],
Percentage: "",
})
} else {
// Percentage is available
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: "in_progress",
File: splitted[0],
Percentage: splitted[1],
})
}
s.downloadMutex.Lock()
status := s.downloadStatus
file := s.downloadFile
percentage := s.downloadPercentage
s.downloadMutex.Unlock()
return
}
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: "idle",
File: "",
Percentage: "",
Status: string(status),
File: file,
Percentage: percentage,
})
}
func (s *Service) DownloadImageFile(c *gin.Context) {
var rsp proto.Response
log.Debug("DownloadImage")
// Set a sentinel file to mark that there is a download in progress
// This is to prevent multiple downloads at the same time
if _, err := os.Stat(sentinelPath); err == nil {
log.Debug("Download in progress")
rsp.ErrRsp(c, -1, "download in progress")
return
}
// Create the sentinel file
err := os.WriteFile(sentinelPath, []byte("start"), 0644)
log.Debug("DownloadImageFile")
expectedSHA256, err := parseSHA256(c.GetHeader("X-SHA256-Sum"))
if err != nil {
log.Error("Failed to create sentinel file")
rsp.ErrRsp(c, -1, "failed to create sentinel file")
rsp.ErrRsp(c, -1, err.Error())
return
}
// Multipart Reader direkt nutzen (keine FormFile!)
reader, err := c.Request.MultipartReader()
if err != nil {
done, err := s.beginDownload("", nil)
if err != nil {
rsp.ErrRsp(c, -1, err.Error())
return
}
finalStatus := downloadStatusFailed
defer func() {
s.finishDownload(done, finalStatus)
}()
reader, err := c.Request.MultipartReader()
if err != nil {
log.Error("invalid multipart data")
rsp.ErrRsp(c, -1, "invalid multipart data")
defer os.Remove(sentinelPath)
return
}
return
}
var lw *loggingWriter
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
log.Error("failed to read part")
rsp.ErrRsp(c, -1, "failed to read part")
lw.stopTicker()
defer os.Remove(sentinelPath)
return
}
if part.FormName() != "file" {
continue
}
filename := part.FileName()
if filename == "" {
log.Error("no filename")
rsp.ErrRsp(c, -1, "no filename")
lw.stopTicker()
defer os.Remove(sentinelPath)
return
}
filename = filepath.Base(filename)
if filename != part.FileName() {
log.Warn("path detected in filename")
rsp.ErrRsp(c, -1, "invalid filename")
defer os.Remove(sentinelPath)
return
}
if strings.Contains(filename, "..") {
log.Warn("path traversal attempt")
rsp.ErrRsp(c, -1, "invalid filename")
defer os.Remove(sentinelPath)
return
if part.FormName() != "file" {
_ = part.Close()
continue
}
if !strings.HasSuffix(strings.ToLower(filename), ".iso") {
rsp.ErrRsp(c, -1, "only .iso files allowed")
defer os.Remove(sentinelPath)
filename := part.FileName()
if err := validateISOFilename(filename); err != nil {
_ = part.Close()
rsp.ErrRsp(c, -1, err.Error())
return
}
s.setDownloadFile(done, filename)
valid := regexp.MustCompile(`^[a-zA-Z0-9._-]+$`)
if !valid.MatchString(filename) {
rsp.ErrRsp(c, -1, "invalid filename")
defer os.Remove(sentinelPath)
return
}
data, err := os.ReadFile(sentinelPath)
out, err := os.CreateTemp("/data", ".nanokvm-upload-*")
if err != nil {
log.Error("Read failed")
rsp.ErrRsp(c, -1, "Read failed")
lw.stopTicker()
defer os.Remove(sentinelPath)
_ = part.Close()
log.Error("cannot create temporary file")
rsp.ErrRsp(c, -1, "cannot create temporary file")
return
}
tempPath := out.Name()
defer os.Remove(tempPath)
outPath := "/data/" + filename
out, err := os.Create(outPath)
if err != nil {
log.Error("cannot create file")
rsp.ErrRsp(c, -1, "cannot create file")
lw.stopTicker()
defer os.Remove(sentinelPath)
return
}
defer out.Close()
if strings.Contains(string(data), "start") {
err = os.WriteFile(sentinelPath, []byte(filename), 0644)
if err != nil {
log.Error("Failed to create sentinel file")
rsp.ErrRsp(c, -1, "failed to create sentinel file")
lw.stopTicker()
defer os.Remove(outPath)
defer os.Remove(sentinelPath)
return
}
lw = &loggingWriter{writer: out, totalSize: c.Request.ContentLength}
lw.startTicker()
} else {
if !strings.Contains(string(data), filename) {
log.Error("failed")
rsp.ErrRsp(c, -1, "failed")
lw.stopTicker()
defer os.Remove(outPath)
defer os.Remove(sentinelPath)
return
}
}
// Direkt streamen → kein RAM-Bedarf außer kleinem Buffer
_, err = io.Copy(lw, part)
if err != nil {
hasher := sha256.New()
lw := newLoggingWriter(io.MultiWriter(out, hasher), c.Request.ContentLength, func(percentage string) {
s.setDownloadProgress(done, percentage)
})
_, copyErr := io.Copy(lw, part)
lw.stopTicker()
partCloseErr := part.Close()
outCloseErr := out.Close()
if copyErr != nil || partCloseErr != nil || outCloseErr != nil {
log.Error("write failed")
rsp.ErrRsp(c, -1, "write failed")
lw.stopTicker()
defer os.Remove(outPath)
defer os.Remove(sentinelPath)
return
}
ok, err := isISO9660(outPath)
if err != nil || !ok {
rsp.ErrRsp(c, -1, "file is not a valid ISO image")
lw.stopTicker()
defer os.Remove(outPath)
defer os.Remove(sentinelPath)
return
}
}
lw.stopTicker()
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: "idle",
File: "",
Percentage: "",
})
defer os.Remove(sentinelPath)
return
if expectedSHA256 != nil && !bytes.Equal(hasher.Sum(nil), expectedSHA256) {
finalStatus = downloadStatusChecksumFailed
rsp.ErrRsp(c, -1, errSHA256Mismatch.Error())
return
}
valid, err := isISO9660(tempPath)
if err != nil || !valid {
rsp.ErrRsp(c, -1, "file is not a valid ISO image")
return
}
outPath := filepath.Join("/data", filename)
if err := os.Rename(tempPath, outPath); err != nil {
rsp.ErrRsp(c, -1, "cannot install uploaded image")
return
}
finalStatus = downloadStatusIdle
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: string(downloadStatusIdle),
File: "",
Percentage: "",
})
return
}
rsp.ErrRsp(c, -1, "file is required")
}
func validateISOFilename(filename string) error {
if filename == "" {
return errors.New("no filename")
}
if filepath.Base(filename) != filename || strings.Contains(filename, "..") {
return errors.New("invalid filename")
}
if !strings.HasSuffix(strings.ToLower(filename), ".iso") {
return errors.New("only .iso files allowed")
}
if !validISOFilename.MatchString(filename) {
return errors.New("invalid filename")
}
return nil
}
func (s *Service) DownloadImage(c *gin.Context) {
var req proto.MountImageReq
var req proto.DownloadImageReq
var rsp proto.Response
log.Debug("DownloadImage")
@@ -291,127 +330,190 @@ func (s *Service) DownloadImage(c *gin.Context) {
return
}
if req.File == "" {
rsp.ErrRsp(c, -1, "invalid arguments")
expectedSHA256, err := parseSHA256(req.SHA256Sum)
if err != nil {
rsp.ErrRsp(c, -1, err.Error())
return
}
// Parse the URI to see if its valid http/s
u, err := url.Parse(req.File)
if err != nil || u.Scheme == "" || u.Host == "" {
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.Path == "" {
rsp.ErrRsp(c, -1, "invalid url")
return
}
filename := filepath.Base(u.Path)
if filename == "." || filename == "/" || filename == "" {
rsp.ErrRsp(c, -1, "invalid url")
return
}
// Set a sentinel file to mark that there is a download in progress
// This is to prevent multiple downloads at the same time
if _, err := os.Stat(sentinelPath); err == nil {
log.Debug("Download in progress")
rsp.ErrRsp(c, -1, "download in progress")
return
}
// Create the sentinel file
err = os.WriteFile(sentinelPath, []byte(req.File), 0644)
ctx, cancel := context.WithCancel(context.Background())
done, err := s.beginDownload(req.File, cancel)
if err != nil {
log.Error("Failed to create sentinel file")
rsp.ErrRsp(c, -1, "failed to create sentinel file")
cancel()
rsp.ErrRsp(c, -1, err.Error())
return
}
// Check if it actually exists and fail if it doesn't
resp, err := http.Head(req.File)
if resp.StatusCode != http.StatusOK || err != nil {
rsp.ErrRsp(c, resp.StatusCode, "failed when checking the url")
log.Error("Failed to check the URL")
defer os.Remove(sentinelPath)
return
}
defer resp.Body.Close()
// Download the image in a goroutine to not block the request
go func() {
defer os.Remove(sentinelPath)
resp, err = http.Get(req.File)
if err != nil {
log.Error("Failed to download the file")
rsp.ErrRsp(c, -1, "failed to download the file")
return
}
defer resp.Body.Close()
// Create the destination file
destPath := filepath.Join("/data", filepath.Base(u.Path))
out, err := os.Create(destPath)
if err != nil {
log.Error("Failed to create destination file")
rsp.ErrRsp(c, -1, "failed to create destination file")
return
}
defer out.Close()
defer cancel()
lw := &loggingWriter{writer: out, totalSize: resp.ContentLength}
lw.startTicker()
_, err = io.Copy(lw, resp.Body)
if err != nil {
log.Error("Failed to save the file")
rsp.ErrRsp(c, -1, "failed to save the file")
lw.stopTicker()
if err := s.downloadRemoteImage(ctx, req.File, expectedSHA256, filename, func(percentage string) {
s.setDownloadProgress(done, percentage)
}); err != nil {
if errors.Is(err, context.Canceled) {
log.Debug("Image download canceled")
s.finishDownload(done, downloadStatusIdle)
return
}
log.Errorf("Failed to download image: %v", err)
status := downloadStatusFailed
if errors.Is(err, errSHA256Mismatch) {
status = downloadStatusChecksumFailed
}
s.finishDownload(done, status)
return
}
lw.stopTicker()
s.finishDownload(done, downloadStatusSuccess)
}()
rsp.OkRspWithData(c, &proto.StatusImageRsp{
Status: "in_progress",
Status: string(downloadStatusInProgress),
File: req.File,
Percentage: "",
})
}
func parseSHA256(value string) ([]byte, error) {
value = strings.TrimSpace(value)
if value == "" {
return nil, nil
}
sum, err := hex.DecodeString(value)
if err != nil || len(sum) != sha256.Size {
return nil, errors.New("invalid sha256sum")
}
return sum, nil
}
func (s *Service) downloadRemoteImage(
ctx context.Context,
rawURL string,
expectedSHA256 []byte,
filename string,
onProgress func(string),
) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return fmt.Errorf("create download request failed: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("download request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download request returned status %d", resp.StatusCode)
}
tempFile, err := os.CreateTemp("/data", ".nanokvm-download-*")
if err != nil {
return fmt.Errorf("create temporary image failed: %w", err)
}
tempPath := tempFile.Name()
defer os.Remove(tempPath)
hasher := sha256.New()
lw := newLoggingWriter(io.MultiWriter(tempFile, hasher), resp.ContentLength, onProgress)
_, copyErr := io.Copy(lw, resp.Body)
lw.stopTicker()
closeErr := tempFile.Close()
if ctx.Err() != nil {
return ctx.Err()
}
if copyErr != nil {
return fmt.Errorf("save downloaded image failed: %w", copyErr)
}
if closeErr != nil {
return fmt.Errorf("close downloaded image failed: %w", closeErr)
}
if expectedSHA256 != nil && !bytes.Equal(hasher.Sum(nil), expectedSHA256) {
return errSHA256Mismatch
}
if ctx.Err() != nil {
return ctx.Err()
}
destPath := filepath.Join("/data", filename)
if err := os.Rename(tempPath, destPath); err != nil {
return fmt.Errorf("install downloaded image failed: %w", err)
}
return nil
}
type loggingWriter struct {
writer io.Writer
total int64
totalSize int64
ticker *time.Ticker
done chan bool
writer io.Writer
total atomic.Int64
totalSize int64
ticker *time.Ticker
done chan struct{}
stopOnce sync.Once
onProgress func(string)
}
func newLoggingWriter(writer io.Writer, totalSize int64, onProgress func(string)) *loggingWriter {
lw := &loggingWriter{
writer: writer,
totalSize: totalSize,
onProgress: onProgress,
}
lw.startTicker()
return lw
}
func (lw *loggingWriter) startTicker() {
lw.ticker = time.NewTicker(2500 * time.Millisecond)
lw.done = make(chan bool)
lw.done = make(chan struct{})
go func() {
for {
select {
case <-lw.done:
return
case <-lw.ticker.C:
lw.updateSentinel()
lw.updateProgress()
}
}
}()
}
func (lw *loggingWriter) stopTicker() {
lw.ticker.Stop()
lw.done <- true
if lw == nil {
return
}
lw.stopOnce.Do(func() {
lw.ticker.Stop()
close(lw.done)
})
}
func (lw *loggingWriter) updateSentinel() {
percentage := float64(lw.total) / float64(lw.totalSize) * 100
content, err := os.ReadFile(sentinelPath)
if err != nil {
log.Error("Failed to read sentinel file")
func (lw *loggingWriter) updateProgress() {
if lw.totalSize <= 0 || lw.onProgress == nil {
return
}
splitted := strings.Split(string(content), ";")
if len(splitted) == 0 {
return
}
err = os.WriteFile(sentinelPath, []byte(fmt.Sprintf("%s;%.2f%%", splitted[0], percentage)), 0644)
if err != nil {
log.Error("Failed to update sentinel file")
}
percentage := float64(lw.total.Load()) / float64(lw.totalSize) * 100
lw.onProgress(fmt.Sprintf("%.2f%%", percentage))
}
func (lw *loggingWriter) Write(p []byte) (int, error) {
n, err := lw.writer.Write(p)
lw.total += int64(n)
lw.total.Add(int64(n))
return n, err
}