Replace Paste.Content field by P.URL and FU.PasteID

Reviewed-on: dsprenkels/rushlink#72

handlers.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
-	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 )
@@ -94,7 +93,6 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request
 	vars := mux.Vars(r)
 	key := vars["key"]
 	var p *db.Paste
-	var fu *db.FileUpload
 	var notFound bool
 	err := rl.db.Transaction(func(tx *db.Database) error {
 		var err error
@@ -102,14 +100,6 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request
 		if err != nil {
 			return err
 		}
-		if p != nil && p.Type == db.PasteTypeFileUpload {
-			var id uuid.UUID
-			copy(id[:], p.Content)
-			fu, err = db.GetFileUpload(tx, id)
-			if err != nil {
-				return err
-			}
-		}
 		return nil
 	})
 	if notFound {
@@ -123,12 +113,12 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request
 		rl.renderError(w, r, status, err.Error())
 		return
 	}
-	rl.viewPasteHandlerInner(w, r, flags, p, fu)
+	rl.viewPasteHandlerInner(w, r, flags, p)
 }
 
-func (rl *rushlink) viewPasteHandlerInner(w http.ResponseWriter, r *http.Request, flags viewPaste, p *db.Paste, fu *db.FileUpload) {
+func (rl *rushlink) viewPasteHandlerInner(w http.ResponseWriter, r *http.Request, flags viewPaste, p *db.Paste) {
 	if flags&viewShowMeta != 0 {
-		rl.viewPasteHandlerInnerMeta(w, r, p, fu)
+		rl.viewPasteHandlerInnerMeta(w, r, p)
 		return
 	}
 
@@ -136,10 +126,7 @@ func (rl *rushlink) viewPasteHandlerInner(w http.ResponseWriter, r *http.Request
 	case db.PasteStatePresent:
 		switch p.Type {
 		case db.PasteTypeFileUpload:
-			if fu == nil {
-				panic(fmt.Sprintf("file for id %v does not exist in database\n", string(p.Content)))
-			}
-			rl.viewFileUploadHandler(w, r, fu)
+			rl.viewFileUploadHandler(w, r, p.FileUpload)
 			return
 		case db.PasteTypeRedirect:
 			if flags&viewNoRedirect != 0 {
@@ -191,7 +178,7 @@ func (rl *rushlink) viewFileUploadHandler(w http.ResponseWriter, r *http.Request
 	http.ServeContent(w, r, fu.FileName, modtime, file)
 }
 
-func (rl *rushlink) viewPasteHandlerInnerMeta(w http.ResponseWriter, r *http.Request, p *db.Paste, fu *db.FileUpload) {
+func (rl *rushlink) viewPasteHandlerInnerMeta(w http.ResponseWriter, r *http.Request, p *db.Paste) {
 	var cd canDelete
 	deleteToken := getDeleteTokenFromRequest(r)
 	if deleteToken != "" {
@@ -203,8 +190,8 @@ func (rl *rushlink) viewPasteHandlerInnerMeta(w http.ResponseWriter, r *http.Req
 	}
 
 	var fileExt string
-	if fu != nil {
-		fileExt = fu.Ext()
+	if p.FileUpload != nil {
+		fileExt = p.FileUpload.Ext()
 	}
 	data := map[string]interface{}{
 		"Paste":           p,
@@ -290,7 +277,7 @@ func (rl *rushlink) newFileUploadPasteHandler(w http.ResponseWriter, r *http.Req
 			panic(errors.Wrap(err, "saving fileUpload in db"))
 		}
 
-		paste, err = shortenFileUploadID(tx, fu.PubID)
+		paste, err = shortenFileUploadID(tx, fu)
 		return err
 	}); err != nil {
 		panic(err)
@@ -370,25 +357,39 @@ func (rl *rushlink) deletePasteHandler(w http.ResponseWriter, r *http.Request) {
 //
 // Returns the new paste key if the fileUpload was successfully added to the
 // database
-func shortenFileUploadID(tx *db.Database, id uuid.UUID) (*db.Paste, error) {
-	return shorten(tx, db.PasteTypeFileUpload, id[:])
+func shortenFileUploadID(tx *db.Database, fu *db.FileUpload) (*db.Paste, error) {
+	// Generate the paste key
+	pasteKey, err := db.GeneratePasteKey(tx, highOnlineEntropy)
+	if err != nil {
+		return nil, errors.Wrap(err, "generating paste key")
+	}
+
+	// Also generate a deleteToken
+	deleteToken, err := db.GenerateDeleteToken()
+	if err != nil {
+		return nil, errors.Wrap(err, "generating delete token")
+	}
+
+	// Store the new
+	p := db.Paste{
+		Type:        db.PasteTypeFileUpload,
+		State:       db.PasteStatePresent,
+		FileUpload:  fu,
+		Key:         pasteKey,
+		DeleteToken: deleteToken,
+	}
+	if err := p.Save(tx); err != nil {
+		return nil, err
+	}
+	return &p, nil
 }
 
 // Add a new URL to the database
 //
 // Returns the new paste key if the url was successfully shortened
 func shortenURL(tx *db.Database, userURL *url.URL) (*db.Paste, error) {
-	return shorten(tx, db.PasteTypeRedirect, []byte(userURL.String()))
-}
-
-// Add a paste (of any kind) to the database with arbitrary content.
-func shorten(tx *db.Database, ty db.PasteType, content []byte) (*db.Paste, error) {
 	// Generate the paste key
-	var keyEntropy int
-	if ty == db.PasteTypeFileUpload || ty == db.PasteTypePaste {
-		keyEntropy = highOnlineEntropy
-	}
-	pasteKey, err := db.GeneratePasteKey(tx, keyEntropy)
+	pasteKey, err := db.GeneratePasteKey(tx, 0)
 	if err != nil {
 		return nil, errors.Wrap(err, "generating paste key")
 	}
@@ -401,9 +402,9 @@ func shorten(tx *db.Database, ty db.PasteType, content []byte) (*db.Paste, error
 
 	// Store the new key
 	p := db.Paste{
-		Type:        ty,
+		Type:        db.PasteTypeRedirect,
 		State:       db.PasteStatePresent,
-		Content:     content,
+		URL:         userURL.String(),
 		Key:         pasteKey,
 		DeleteToken: deleteToken,
 	}

internal/db/db.go

@@ -35,7 +35,11 @@ var (
 		LogLevel:      logger.Warn,
 		Colorful:      true,
 	})
-	gormConfig = gorm.Config{Logger: gormLogger, PrepareStmt: true}
+	gormConfig = gorm.Config{
+		Logger:                                   gormLogger,
+		PrepareStmt:                              true,
+		DisableForeignKeyConstraintWhenMigrating: true,
+	}
 )
 
 // OpenDBFromEnvironment tries to open an SQL database, described by

internal/db/fileupload.go

@@ -38,6 +38,8 @@ type FileUpload struct {
 	// UUID publically identifies this FileUpload.
 	PubID uuid.UUID `gorm:"uniqueIndex"`
 
+	PasteID uint
+
 	// FileName contains the original filename of this FileUpload.
 	FileName string
 

internal/db/migrate.go

@@ -38,8 +38,69 @@ func Gormigrate(db *gorm.DB) *gormigrate.Gormigrate {
 				}
 				return tx.AutoMigrate(&FileUpload{}, &Paste{})
 			},
-			Rollback: func(tx *gorm.DB) error {
-				return tx.Migrator().DropTable(&FileUpload{}, &Paste{})
+		},
+		{
+			ID: "202107231722",
+			Migrate: func(tx *gorm.DB) error {
+				// Update the schema
+				type FileUpload struct {
+					ID          uint            `gorm:"primaryKey"`
+					State       FileUploadState `gorm:"index"`
+					PubID       uuid.UUID       `gorm:"uniqueIndex"`
+					PasteID     uint
+					FileName    string
+					ContentType string
+					Checksum    uint32
+					CreatedAt   time.Time
+					UpdatedAt   time.Time
+					DeletedAt   gorm.DeletedAt
+				}
+				type Paste struct {
+					ID          uint       `gorm:"primaryKey"`
+					Type        PasteType  `gorm:"index"`
+					State       PasteState `gorm:"index"`
+					Content     []byte
+					URL         string
+					FileUpload  *FileUpload
+					Key         string `gorm:"uniqueIndex"`
+					DeleteToken string
+					CreatedAt   time.Time
+					UpdatedAt   time.Time
+					DeletedAt   gorm.DeletedAt
+				}
+				err := tx.AutoMigrate(&FileUpload{}, &Paste{})
+				if err != nil {
+					return err
+				}
+
+				// Migrate the data
+				pastes := make([]Paste, 0)
+				if err := tx.Model(&Paste{}).Find(&pastes).Error; err != nil {
+					return err
+				}
+				for _, p := range pastes {
+					switch p.Type {
+					case PasteTypeRedirect:
+						p.URL = string(p.Content)
+						tx.Model(p).Select("Content", "URL").Updates(p)
+					case PasteTypeFileUpload:
+						var id uuid.UUID
+						var fus []FileUpload
+						copy(id[:], p.Content)
+						if err := db.Unscoped().Limit(1).Where("pub_id = ?", id).Find(&fus).Error; err != nil {
+							return err
+						}
+						p.FileUpload = &fus[0]
+						tx.Model(p).Select("Content", "FileUpload").Updates(p)
+					default:
+						continue
+					}
+				}
+
+				// Currently there is a bug in GORM, which causes a nil ptr
+				// dereference when you try dropping a column in an sqlite3
+				// database.
+				return nil
 			},
 		},
 	})

internal/db/paste.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 )
 
 // PasteType describes the type of Paste (i.e. file, redirect, [...]).
@@ -24,7 +24,8 @@ type Paste struct {
 	ID          uint       `gorm:"primaryKey"`
 	Type        PasteType  `gorm:"index"`
 	State       PasteState `gorm:"index"`
-	Content     []byte
+	URL         string
+	FileUpload  *FileUpload
 	Key         string `gorm:"uniqueIndex"`
 	DeleteToken string
 	CreatedAt   time.Time
@@ -146,7 +147,7 @@ func ValidatePasteKey(key string) error {
 // the key format first.
 func GetPasteNoValidate(db *gorm.DB, key string) (*Paste, error) {
 	var ps []Paste
-	if err := db.Unscoped().Limit(1).Where("key = ?", key).Find(&ps).Error; err != nil {
+	if err := db.Unscoped().Preload(clause.Associations).Where("key = ?", key).Find(&ps).Error; err != nil {
 		return nil, err
 	}
 	if len(ps) == 0 {
@@ -163,16 +164,8 @@ func (p *Paste) Save(db *gorm.DB) error {
 // Delete deletes this Paste from the database.
 func (p *Paste) Delete(db *gorm.DB, fs *FileStore) error {
 	// Remove the (maybe) attached file
-	if p.Type == PasteTypeFileUpload {
-		fuID, err := uuid.FromBytes(p.Content)
-		if err != nil {
-			return errors.Wrap(err, "failed to parse uuid")
-		}
-		fu, err := GetFileUpload(db, fuID)
-		if err != nil {
-			return errors.Wrap(err, "failed to find file in database")
-		}
-		if err := fu.Delete(db, fs); err != nil {
+	if p.FileUpload != nil {
+		if err := p.FileUpload.Delete(db, fs); err != nil {
 			return errors.Wrap(err, "failed to remove file")
 		}
 	}
@@ -180,7 +173,9 @@ func (p *Paste) Delete(db *gorm.DB, fs *FileStore) error {
 	// Wipe the old paste
 	p.Type = PasteTypeUndef
 	p.State = PasteStateDeleted
-	p.Content = []byte{}
+	p.URL = ""
+	p.FileUpload = nil
+
 	if err := db.Save(&p).Error; err != nil {
 		return errors.Wrap(err, "failed to wipe paste in database")
 	}
@@ -199,10 +194,9 @@ func (p *Paste) RedirectURL() *url.URL {
 	if p.Type != PasteTypeRedirect {
 		panic("expected p.Type to be PasteTypeRedirect")
 	}
-	rawurl := string(p.Content)
-	urlParse, err := url.Parse(rawurl)
+	urlParse, err := url.Parse(p.URL)
 	if err != nil {
-		panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key))
+		panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", p.URL, p.Key))
 	}
 	return urlParse
 }

metrics.go

@@ -14,42 +14,59 @@ import (
 
 const metricNamespace = "rushlink"
 
-var metricRequestsTotalCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+// metricURLsTotalGauge counts the number of requests that are handled by
+// the application, partitioned by status code and HTTP method.
+//
+// This counter is updated by the router.
+var metricRequestsTotalCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
 		Namespace: metricNamespace,
 		Subsystem: "http",
 		Name:      "requests_total",
 		Help:      "How many HTTP requests processed, partitioned by status code and HTTP method.",
-}, []string{"code", "method"})
+	}, []string{"code", "method"})
 
-func metricURLsTotal(database *db.Database) float64 {
-	var metric float64
-	if err := database.Transaction(func(tx *db.Database) error {
-		var count int64
-		if err := database.Model(&db.Paste{}).Count(&count).Error; err != nil {
-			return err
-		}
-		metric = float64(count)
-		return nil
-	}); err != nil {
-		log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric"))
-		return 0
-	}
-	return metric
-}
+// metricRequestsLatencyNanoSeconds keeps track of the request latencies for
+// each http request.
+//
+// This historogram is updated by the router.
+var metricRequestsLatencyNanoSeconds = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Namespace: metricNamespace,
+		Subsystem: "http",
+		Name:      "request_duration_seconds",
+		Buckets: []float64{
+			float64(500e-6),
+			float64(1e-3),
+			float64(2e-3),
+			float64(5e-3),
+			float64(10e-3),
+			float64(20e-3),
+			float64(50e-3),
+		},
+		Help: "The latency of each HTTP request, partitioned by status code and HTTP method.",
+	}, []string{"code", "method"})
+
+// metricURLsTotalGauge measures the amount of pastes stored in the database,
+// partitioned by type and state.
+//
+// Its values are computed on the fly by updateMetrics().
+var metricURLsTotalGauge = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: metricNamespace,
+		Subsystem: "pastes",
+		Name:      "urls_total",
+		Help:      "The current amount of pastes in the database, partitioned by state and type.",
+	}, []string{"state", "type"})
 
 // StartMetricsServer starts sering Prometheus metrics exports on addr
 func StartMetricsServer(addr string, database *db.Database, fs *db.FileStore) {
 	prometheus.MustRegister(metricRequestsTotalCounter)
-
-	prometheus.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
-		Namespace: metricNamespace,
-		Subsystem: "pastes",
-		Name:      "urls_total",
-		Help:      "The current amount of pastes in the database.",
-	}, func() float64 { return metricURLsTotal(database) }))
+	prometheus.MustRegister(metricRequestsLatencyNanoSeconds)
+	prometheus.MustRegister(metricURLsTotalGauge)
 
 	router := mux.NewRouter()
-	router.Handle("/metrics", promhttp.Handler()).Methods("GET")
+	router.Handle("/metrics", &MetricsHandler{database}).Methods("GET")
 	srv := &http.Server{
 		Handler:      router,
 		Addr:         addr,
@@ -58,3 +75,34 @@ func StartMetricsServer(addr string, database *db.Database, fs *db.FileStore) {
 	}
 	log.Fatal(srv.ListenAndServe())
 }
+
+type MetricsHandler struct {
+	db *db.Database
+}
+
+func (mh *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	mh.updateMetrics()
+	promhttp.Handler().ServeHTTP(w, r)
+}
+
+func (mh *MetricsHandler) updateMetrics() {
+	// Update metricURLsTotalGauge
+	results := make([](struct {
+		Type  db.PasteType
+		State db.PasteState
+		Count float64
+	}), 0)
+	query := mh.db.Unscoped().Model(&db.Paste{}).Select("type", "state", "COUNT(*) as count").Group("type, state").Find(&results)
+	if err := query.Error; err != nil {
+		log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric"))
+		return
+	}
+	metricURLsTotalGauge.Reset()
+	for _, r := range results {
+		labels := map[string]string{
+			"type":  r.Type.String(),
+			"state": r.State.String(),
+		}
+		metricURLsTotalGauge.With(labels).Set(r.Count)
+	}
+}

router.go

@@ -58,10 +58,18 @@ func (rl *rushlink) recoveryMiddleware(next http.Handler) http.Handler {
 
 func (rl *rushlink) metricsMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tick := time.Now()
 		srw := statusResponseWriter{Inner: w}
 		next.ServeHTTP(&srw, r)
+		tock := time.Now()
+
 		status := strconv.Itoa(srw.StatusCode)
-		metricRequestsTotalCounter.WithLabelValues(status, r.Method).Inc()
+		labels := map[string]string{"code": status, "method": r.Method}
+		// Update requests counter metric
+		metricRequestsTotalCounter.With(labels).Inc()
+		// Update request latency metric
+		elapsed := tock.Sub(tick)
+		metricRequestsLatencyNanoSeconds.With(labels).Observe(elapsed.Seconds())
 	})
 }