db: Migrate FileUpload.ContentTypes to auto-detect

cmd/rushlink/main.go

@@ -18,15 +18,15 @@ var (
 func main() {
 	flag.Parse()
 
-	database, err := db.OpenDB(*databasePath)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	defer database.Close()
 	filestore, err := db.OpenFileStore(*fileStorePath)
 	if err != nil {
 		log.Fatalln(err)
 	}
+	database, err := db.OpenDB(*databasePath, filestore)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer database.Close()
 
 	go rushlink.StartMetricsServer(*metricsListen, database, filestore)
 	rushlink.StartMainServer(*httpListen, database, filestore)

internal/db/db.go

@@ -1,8 +1,12 @@
 package db
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
+	"os"
 	"time"
 
 	"github.com/pkg/errors"
@@ -24,7 +28,7 @@ type Database struct {
 //
 // If we alter the database format, we bump this number and write a new
 // database migration in migrate().
-const CurrentMigrateVersion = 2
+const CurrentMigrateVersion = 3
 
 // BucketConf holds the name for the "configuration" bucket.
 //
@@ -42,7 +46,7 @@ const BucketFileUpload = "fileUpload"
 const KeyMigrateVersion = "migrate_version"
 
 // OpenDB opens a database file located at path.
-func OpenDB(path string) (*Database, error) {
+func OpenDB(path string, fs *FileStore) (*Database, error) {
 	if path == "" {
 		return nil, errors.New("database not set")
 	}
@@ -51,7 +55,7 @@ func OpenDB(path string) (*Database, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to open database at '%v'", path)
 	}
-	if err := db.Update(migrate); err != nil {
+	if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil {
 		return nil, err
 	}
 	return &Database{db}, nil
@@ -66,7 +70,12 @@ func (db *Database) Close() error {
 }
 
 // Initialize and migrate the database to the current version
-func migrate(tx *bolt.Tx) error {
+func migrate(tx *bolt.Tx, fs *FileStore) error {
+	// Guidelines for error handling:
+	//   - Errors based on malformed *structure* should be fatal!
+	//   - Errors based on malformed *data* should print a warning
+	//     (and if possible try to fix the error).
+
 	dbVersion, err := dbVersion(tx)
 	if err != nil {
 		return err
@@ -104,6 +113,54 @@ func migrate(tx *bolt.Tx) error {
 		}
 	}
 
+	if dbVersion < 3 {
+		log.Println("migrating database to version 3")
+		// In this version, we changed te way how Content-Types are being
+		// stored.  Previously, we allowed clients to provide their own
+		// Content-Types for files, using the Content-Disposition header in
+		// multipart forms.  The new way detects these types using
+		// http.DetectContentType.
+		//
+		// Scan through all the FileUploads and update their ContentTypes.
+		bucket := tx.Bucket([]byte(BucketFileUpload))
+		cursor := bucket.Cursor()
+		var id, storedBytes []byte
+		id, storedBytes = cursor.First()
+		for id != nil {
+			fu, err := decodeFileUpload(storedBytes)
+			if err != nil {
+				log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id))
+				id, storedBytes = cursor.Next()
+				continue
+			}
+			if fu.State != FileUploadStatePresent {
+				id, storedBytes = cursor.Next()
+				continue
+			}
+			filePath := fu.Path(fs)
+			file, err := os.Open(fu.Path(fs))
+			if err != nil {
+				log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath))
+				id, storedBytes = cursor.Next()
+				continue
+			}
+			var buf bytes.Buffer
+			buf.Grow(512)
+			io.CopyN(&buf, file, 512)
+			contentType := http.DetectContentType(buf.Bytes())
+			if contentType != fu.ContentType {
+				fu.ContentType = contentType
+				fu.Save(tx)
+				cursor.Seek(id)
+			}
+			id, storedBytes = cursor.Next()
+		}
+		// Update the version number
+		if err := setDBVersion(tx, 3); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

internal/db/fileupload.go

@@ -153,6 +153,10 @@ func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) {
 	if storedBytes == nil {
 		return nil, nil
 	}
+	return decodeFileUpload(storedBytes)
+}
+
+func decodeFileUpload(storedBytes []byte) (*FileUpload, error) {
 	fu := &FileUpload{}
 	err := gobmarsh.Unmarshal(storedBytes, fu)
 	return fu, err