diff --git a/cmd/rushlink-migrate-db/main.go b/cmd/rushlink-migrate-db/main.go deleted file mode 100644 index 35f0f53..0000000 --- a/cmd/rushlink-migrate-db/main.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "flag" - "log" - "time" - - "gitea.hashru.nl/dsprenkels/rushlink/internal/boltdb" - "gitea.hashru.nl/dsprenkels/rushlink/internal/db" - bolt "go.etcd.io/bbolt" - "gorm.io/gorm" -) - -var ( - boltDBPath = flag.String("boltdb", "", "location of the bolt database file") - fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored") -) - -func main() { - flag.Parse() - - boltFileStore, err := boltdb.OpenFileStore(*fileStorePath) - if err != nil { - log.Fatalln(err) - } - boltDB, err := boltdb.OpenDB(*boltDBPath, boltFileStore) - if err != nil { - log.Fatalln(err) - } - defer boltDB.Close() - sqlDB, err := db.OpenDBFromEnvironment() - if err != nil { - log.Fatalln(err) - } - - // Migrate database schema - m := db.Gormigrate(sqlDB) - if err := m.MigrateTo("202010251337"); err != nil { - log.Fatalln(err) - } - - // Migrate all files in filestorage - if err := sqlDB.Transaction(func(sqlTx *gorm.DB) error { - return boltDB.Bolt.View(func(boltTx *bolt.Tx) error { - // Migrate all the file uploads - allFUs, err := boltdb.AllFileUploads(boltTx) - if err != nil { - return err - } - var fusDeleted uint - var sqlFUs []db.FileUpload - for _, fu := range allFUs { - isDeleted := fu.State == boltdb.FileUploadStateDeleted - sqlFU := db.FileUpload{ - State: db.FileUploadState(int(fu.State)), - PubID: fu.ID, - FileName: fu.FileName, - ContentType: fu.ContentType, - Checksum: fu.Checksum, - DeletedAt: deletedAt(isDeleted), - } - sqlFUs = append(sqlFUs, sqlFU) - if isDeleted { - fusDeleted++ - } - } - log.Printf("migrating %v file uploads (of which %v deleted)", len(sqlFUs), fusDeleted) - sqlTx.Create(sqlFUs) - - // Migrate all the pastes. - allPastes, err := boltdb.AllPastes(boltTx) - if err != nil { - return err - } - var pastesDeleted uint - var sqlPastes []db.Paste - for _, paste := range allPastes { - isDeleted := paste.State == boltdb.PasteStateDeleted - sqlPaste := db.Paste{ - Type: db.PasteType(int(paste.Type)), - State: db.PasteState(int(paste.State)), - Content: paste.Content, - Key: paste.Key, - DeleteToken: paste.DeleteToken, - CreatedAt: paste.TimeCreated, - DeletedAt: deletedAt(isDeleted), - } - sqlPastes = append(sqlPastes, sqlPaste) - if isDeleted { - pastesDeleted++ - } - } - log.Printf("migrating %v pastes (of which %v deleted)", len(sqlPastes), pastesDeleted) - sqlTx.Create(sqlPastes) - - return nil - }) - }); err != nil { - log.Fatalln(err) - } - - log.Println("migration successful! :D") -} - -func deletedAt(isDeleted bool) gorm.DeletedAt { - if isDeleted { - return gorm.DeletedAt{Time: time.Now(), Valid: true} - } - return gorm.DeletedAt{} -} diff --git a/go.mod b/go.mod index 3c8e51b..c1db1a8 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/common v0.23.0 // indirect - go.etcd.io/bbolt v1.3.5 golang.org/x/sys v0.0.0-20210507161434-a76c4d0a0096 // indirect gorm.io/driver/postgres v1.1.0 gorm.io/driver/sqlite v1.1.4 diff --git a/go.sum b/go.sum index 7b4767a..d456015 100644 --- a/go.sum +++ b/go.sum @@ -382,8 +382,6 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -471,7 +469,6 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/boltdb/db.go b/internal/boltdb/db.go deleted file mode 100644 index f433d81..0000000 --- a/internal/boltdb/db.go +++ /dev/null @@ -1,204 +0,0 @@ -package boltdb - -import ( - "bytes" - "fmt" - "io" - "log" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - - bolt "go.etcd.io/bbolt" - - gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" -) - -// Database is the main rushlink database type. -// -// Open a database using DB.Open() and close it in the end using DB.Close(). -// Only one instance of DB should exist in a program at any moment. -type Database struct { - Bolt *bolt.DB -} - -// CurrentMigrateVersion holds the current "migrate version". -// -// If we alter the database format, we bump this number and write a new -// database migration in migrate(). -const CurrentMigrateVersion = 3 - -// BucketConf holds the name for the "configuration" bucket. -// -// This bucket holds the database version, secret site-wide keys, etc. -const BucketConf = "conf" - -// BucketPastes holds the name for the pastes bucket. -const BucketPastes = "pastes" - -// BucketFileUpload holds the name for the file-upload bucket. -const BucketFileUpload = "fileUpload" - -// KeyMigrateVersion stores the current migration version. If this value is less than -// CurrentMigrateVersion, the database has to be migrated. -const KeyMigrateVersion = "migrate_version" - -// OpenDB opens a database file located at path. -func OpenDB(path string, fs *FileStore) (*Database, error) { - if path == "" { - return nil, errors.New("database not set") - } - - db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return nil, errors.Wrapf(err, "failed to open database at '%v'", path) - } - if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil { - return nil, err - } - return &Database{db}, nil -} - -// Close the bolt database -func (db *Database) Close() error { - if db == nil { - panic("no open database") - } - return db.Bolt.Close() -} - -// Initialize and migrate the database to the current version -func migrate(tx *bolt.Tx, fs *FileStore) error { - // Guidelines for error handling: - // - Errors based on malformed *structure* should be fatal! - // - Errors based on malformed *data* should print a warning - // (and if possible try to fix the error). - - dbVersion, err := dbVersion(tx) - if err != nil { - return err - } - - // Migrate the database to version 1 - if dbVersion < 1 { - log.Println("migrating database to version 1") - // Create conf bucket - _, err := tx.CreateBucket([]byte(BucketConf)) - if err != nil { - return err - } - // Create paste bucket - _, err = tx.CreateBucket([]byte(BucketPastes)) - if err != nil { - return err - } - // Update the version number - if err := setDBVersion(tx, 1); err != nil { - return err - } - } - - if dbVersion < 2 { - log.Println("migrating database to version 2") - // Create fileUpload bucket - _, err := tx.CreateBucket([]byte(BucketFileUpload)) - if err != nil { - return err - } - // Update the version number - if err := setDBVersion(tx, 2); err != nil { - return err - } - } - - if dbVersion < 3 { - log.Println("migrating database to version 3") - // In this version, we changed te way how Content-Types are being - // stored. Previously, we allowed clients to provide their own - // Content-Types for files, using the Content-Disposition header in - // multipart forms. The new way detects these types using - // http.DetectContentType. - // - // Scan through all the FileUploads and update their ContentTypes. - bucket := tx.Bucket([]byte(BucketFileUpload)) - cursor := bucket.Cursor() - var id, storedBytes []byte - id, storedBytes = cursor.First() - for id != nil { - fu, err := decodeFileUpload(storedBytes) - if err != nil { - log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id)) - id, storedBytes = cursor.Next() - continue - } - if fu.State != FileUploadStatePresent { - id, storedBytes = cursor.Next() - continue - } - filePath := fu.Path(fs) - file, err := os.Open(fu.Path(fs)) - if err != nil { - log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath)) - id, storedBytes = cursor.Next() - continue - } - var buf bytes.Buffer - buf.Grow(512) - io.CopyN(&buf, file, 512) - contentType := http.DetectContentType(buf.Bytes()) - if contentType != fu.ContentType { - fu.ContentType = contentType - fu.Save(tx) - cursor.Seek(id) - } - id, storedBytes = cursor.Next() - } - // Update the version number - if err := setDBVersion(tx, 3); err != nil { - return err - } - } - - return nil -} - -// Get the current migrate version from the database -func dbVersion(tx *bolt.Tx) (int, error) { - conf := tx.Bucket([]byte(BucketConf)) - if conf == nil { - return 0, nil - } - dbVersionBytes := conf.Get([]byte(KeyMigrateVersion)) - if dbVersionBytes == nil { - return 0, nil - } - - // Version was already stored - var dbVersion int - if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil { - return 0, err - } - if dbVersion == 0 { - return 0, fmt.Errorf("database version is invalid (%v)", dbVersion) - } - if dbVersion > CurrentMigrateVersion { - return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion) - } - return dbVersion, nil -} - -// Update the current migrate version in the database -func setDBVersion(tx *bolt.Tx, version int) error { - conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf)) - if err != nil { - return err - } - - versionBytes, err := gobmarsh.Marshal(version) - if err != nil { - return err - } - return conf.Put([]byte(KeyMigrateVersion), versionBytes) -} diff --git a/internal/boltdb/fileupload.go b/internal/boltdb/fileupload.go deleted file mode 100644 index adb2dbf..0000000 --- a/internal/boltdb/fileupload.go +++ /dev/null @@ -1,256 +0,0 @@ -package boltdb - -import ( - "bytes" - "encoding/hex" - "hash/crc32" - "io" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - - "github.com/google/uuid" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" - - gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" -) - -// Use the Castagnoli checksum because of the acceleration on Intel CPUs -var checksumTable = crc32.MakeTable(crc32.Castagnoli) - -// FileStore holds the path to a file storage location. -type FileStore struct { - path string -} - -// FileUploadState determines the current state of a FileUpload object. -type FileUploadState int - -// FileUpload models an uploaded file. -type FileUpload struct { - // State of the FileUpload (present/deleted/etc). - State FileUploadState - - // ID identifies this FileUpload. - ID uuid.UUID - - // FileName contains the original filename of this FileUpload. - FileName string - - // Content type as determined by http.DetectContentType. - ContentType string - - // Checksum holds a crc32c checksum of the file. - // - // This checksum is only meant to allow for the detection of random - // database corruption. - Checksum uint32 -} - -const ( - dirMode os.FileMode = 0750 - fileMode os.FileMode = 0640 -) - -const ( - // FileUploadStateUndef is an undefined FileUpload. - FileUploadStateUndef FileUploadState = 0 - // FileUploadStatePresent denotes the normal (existing) state. - FileUploadStatePresent FileUploadState = 1 - // FileUploadStateDeleted denotes a deleted state. - FileUploadStateDeleted FileUploadState = 2 -) - -func (t FileUploadState) String() string { - switch t { - case FileUploadStateUndef: - return "unknown" - case FileUploadStatePresent: - return "present" - case FileUploadStateDeleted: - return "deleted" - default: - return "invalid" - } -} - -// OpenFileStore opens the file storage at path. -func OpenFileStore(path string) (*FileStore, error) { - if path == "" { - return nil, errors.New("file-store not set") - } - - // Try to create the file store directory if it does not yet exist - if err := os.MkdirAll(path, dirMode); err != nil { - return nil, errors.Wrap(err, "creating file store directory") - } - - return &FileStore{path[:]}, nil -} - -// Path returns the path of the FileStore root. -func (fs *FileStore) Path() string { - return fs.path -} - -// filePath resolves the path of a file in the FileStore given some id and filename. -func (fs *FileStore) filePath(id uuid.UUID, fileName string) string { - if fs.path == "" { - panic("fileStoreDir called while the file store path has not been set") - } - return path.Join(fs.path, hex.EncodeToString(id[:]), fileName) -} - -// NewFileUpload creates a new FileUpload object. -// -// Internally, this function detects the type of the file stored in `r` using -// `http.DetectContentType`. -func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) { - // Generate a file ID - id, err := uuid.NewRandom() - if err != nil { - return nil, errors.Wrap(err, "generating UUID") - } - - // Construct a checksum for this file - hash := crc32.New(checksumTable) - tee := io.TeeReader(r, hash) - - // Detect the file type - var tmpBuf bytes.Buffer - tmpBuf.Grow(512) - io.CopyN(&tmpBuf, tee, 512) - contentType := http.DetectContentType(tmpBuf.Bytes()) - - // Open the file on disk for writing - baseName := filepath.Base(fileName) - filePath := fs.filePath(id, baseName) - if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil { - return nil, errors.Wrap(err, "creating file dir") - } - file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode) - if err != nil { - return nil, errors.Wrap(err, "opening file") - } - defer file.Close() - - // Write the file to disk - _, err = io.Copy(file, &tmpBuf) - if err != nil { - return nil, errors.Wrap(err, "writing to file") - } - _, err = io.Copy(file, tee) - if err != nil { - return nil, errors.Wrap(err, "writing to file") - } - - fu := &FileUpload{ - State: FileUploadStatePresent, - ID: id, - FileName: baseName, - ContentType: contentType, - Checksum: hash.Sum32(), - } - return fu, nil -} - -// GetFileUpload tries to retrieve a FileUpload object from the bolt database. -func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) { - bucket := tx.Bucket([]byte(BucketFileUpload)) - if bucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload) - } - storedBytes := bucket.Get(id[:]) - if storedBytes == nil { - return nil, nil - } - return decodeFileUpload(storedBytes) -} - -// AllFileUploads tries to retrieve all FileUpload objects from the bolt database. -func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) { - bucket := tx.Bucket([]byte(BucketFileUpload)) - if bucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload) - } - var fus []FileUpload - err := bucket.ForEach(func(_, storedBytes []byte) error { - fu, err := decodeFileUpload(storedBytes) - if err != nil { - return err - } - fus = append(fus, *fu) - return nil - }) - if err != nil { - return nil, err - } - return fus, nil -} - -func decodeFileUpload(storedBytes []byte) (*FileUpload, error) { - fu := &FileUpload{} - err := gobmarsh.Unmarshal(storedBytes, fu) - return fu, err -} - -// Save saves a FileUpload in the database. -func (fu *FileUpload) Save(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(BucketFileUpload)) - if bucket == nil { - return errors.Errorf("bucket %v does not exist", BucketFileUpload) - } - - buf, err := gobmarsh.Marshal(fu) - if err != nil { - return errors.Wrap(err, "encoding for database failed") - } - if err := bucket.Put(fu.ID[:], buf); err != nil { - return errors.Wrap(err, "database transaction failed") - } - return nil -} - -// Delete deletes a FileUpload from the database. -func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error { - // Remove the file in the backend - filePath := fu.Path(fs) - if err := os.Remove(filePath); err != nil { - return err - } - - // Update the file in the server - if err := (&FileUpload{ - ID: fu.ID, - State: FileUploadStateDeleted, - }).Save(tx); err != nil { - return err - } - - // Cleanup the parent directory - wrap := "deletion succeeded, but removing the file directory has failed" - return errors.Wrap(os.Remove(path.Dir(filePath)), wrap) -} - -// Path returns the path to this FileUpload in the FileStore provided in fs. -func (fu *FileUpload) Path(fs *FileStore) string { - return fs.filePath(fu.ID, fu.FileName) -} - -// URL returns the URL for the FileUpload. -func (fu *FileUpload) URL() *url.URL { - rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName - urlParse, err := url.Parse(rawurl) - if err != nil { - panic("could not construct /uploads/ url") - } - return urlParse -} - -// Ext returns the extension of the file attached to this FileUpload. -func (fu *FileUpload) Ext() string { - return filepath.Ext(fu.FileName) -} diff --git a/internal/boltdb/paste.go b/internal/boltdb/paste.go deleted file mode 100644 index 50f2194..0000000 --- a/internal/boltdb/paste.go +++ /dev/null @@ -1,345 +0,0 @@ -package boltdb - -import ( - "crypto/rand" - "encoding/base64" - "encoding/hex" - "net/http" - "net/url" - "strings" - "time" - - gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" - "github.com/google/uuid" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" -) - -// PasteType describes the type of Paste (i.e. file, redirect, [...]). -type PasteType int - -// PasteState describes the state of a Paste (i.e. present, deleted, [...]). -type PasteState int - -// Paste describes the main Paste model in the database. -type Paste struct { - Type PasteType - State PasteState - Content []byte - Key string - DeleteToken string - TimeCreated time.Time -} - -// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen -var ReservedPasteKeys = []string{"xd42", "example"} - -// Note: we use iota here. That means removals of PasteType* are not allowed, -// because this changes the value of the constant. Please add the comment -// "// deprecated" if you want to remove the constant. Additions are only -// allowed at the bottom of this block, for the same reason. -const ( - PasteTypeUndef PasteType = iota - // PasteTypePaste is as of yet unused. It is still unclear if this type - // will ever get a proper meaning. - PasteTypePaste - PasteTypeRedirect - PasteTypeFileUpload -) - -// Note: we use iota here. See the comment above PasteType* -const ( - PasteStateUndef PasteState = iota - PasteStatePresent - PasteStateDeleted -) - -// minKeyLen specifies the mimimum length of a paste key. -const minKeyLen = 4 - -var ( - // ErrKeyInvalidChar occurs when a key contains an invalid character. - ErrKeyInvalidChar = errors.New("invalid character in key") - // ErrKeyInvalidLength occurs when a key embeds a length that is incorrect. - ErrKeyInvalidLength = errors.New("key length encoding is incorrect") - // ErrPasteDoesNotExist occurs when a key does not exist in the database. - ErrPasteDoesNotExist = errors.New("url key not found in the database") -) - -// ErrHTTPStatusCode returns the HTTP status code that should correspond to -// the provided error. -// server error, or false if it is not. -func ErrHTTPStatusCode(err error) int { - switch err { - case nil: - return 0 - case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist: - return http.StatusNotFound - } - return http.StatusInternalServerError -} - -// Base64 encoding and decoding -var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" -var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding) - -func (t PasteType) String() string { - switch t { - case PasteTypeUndef: - return "unknown" - case PasteTypePaste: - return "paste" - case PasteTypeRedirect: - return "redirect" - case PasteTypeFileUpload: - return "file" - default: - return "invalid" - } -} - -func (t PasteState) String() string { - switch t { - case PasteStateUndef: - return "unknown" - case PasteStatePresent: - return "present" - case PasteStateDeleted: - return "deleted" - default: - return "invalid" - } -} - -// GetPaste retrieves a paste from the database. -func GetPaste(tx *bolt.Tx, key string) (*Paste, error) { - if err := ValidatePasteKey(key); err != nil { - return nil, err - } - return GetPasteNoValidate(tx, key) -} - -// ValidatePasteKey validates the format of the key that has -func ValidatePasteKey(key string) error { - internalLen := minKeyLen - countingOnes := true - for _, ch := range key { - limb := strings.IndexRune(base64Alphabet, ch) - if limb == -1 { - return ErrKeyInvalidChar - } - for i := 5; i >= 0 && countingOnes; i-- { - if (limb>>uint(i))&0x1 == 0 { - countingOnes = false - break - } - internalLen++ - } - } - if internalLen != len(key) { - return ErrKeyInvalidLength - } - return nil -} - -// GetPasteNoValidate retrieves a paste from the database without validating -// the key format first. -func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) { - pastesBucket := tx.Bucket([]byte(BucketPastes)) - if pastesBucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketPastes) - } - storedBytes := pastesBucket.Get([]byte(key)) - if storedBytes == nil { - return nil, ErrPasteDoesNotExist - } - return decodePaste(storedBytes) -} - -func decodePaste(storedBytes []byte) (*Paste, error) { - p := &Paste{} - err := gobmarsh.Unmarshal(storedBytes, p) - return p, err -} - -// Save saves this Paste to the database. -func (p *Paste) Save(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(BucketPastes)) - if bucket == nil { - return errors.Errorf("bucket %v does not exist", BucketPastes) - } - - buf, err := gobmarsh.Marshal(p) - if err != nil { - return errors.Wrap(err, "encoding for database failed") - } - if err := bucket.Put([]byte(p.Key), buf); err != nil { - return errors.Wrap(err, "database transaction failed") - } - return nil -} - -// Delete deletes this Paste from the database. -func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error { - // Remove the (maybe) attached file - if p.Type == PasteTypeFileUpload { - fuID, err := uuid.FromBytes(p.Content) - if err != nil { - return errors.Wrap(err, "failed to parse uuid") - } - fu, err := GetFileUpload(tx, fuID) - if err != nil { - return errors.Wrap(err, "failed to find file in database") - } - if err := fu.Delete(tx, fs); err != nil { - return errors.Wrap(err, "failed to remove file") - } - } - - // Replace the old paste with a new empty paste - p.Type = PasteTypeUndef - p.State = PasteStateDeleted - p.Content = []byte{} - if err := p.Save(tx); err != nil { - return errors.Wrap(err, "failed to delete paste in database") - } - return nil -} - -// RedirectURL returns the URL from this paste. -// -// This function assumes that the paste is valid. If the paste struct is -// corrupted in some way, this function will panic. -func (p *Paste) RedirectURL() *url.URL { - if p.Type != PasteTypeRedirect { - panic("expected p.Type to be PasteTypeRedirect") - } - rawurl := string(p.Content) - urlParse, err := url.Parse(rawurl) - if err != nil { - panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key)) - } - return urlParse -} - -// GeneratePasteKey generates a new paste key. It will ensure that the newly -// generated paste key does not already exist in the database. -// The running time of this function is in O(log N), where N is the amount of -// keys stored in the url-shorten database. -// In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum -// guessing entropy of the generated key. -func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) { - pastesBucket := tx.Bucket([]byte(BucketPastes)) - if pastesBucket == nil { - return "", errors.Errorf("bucket %v does not exist", BucketPastes) - } - - epoch := 0 - var key string - for { - var err error - key, err = generatePasteKeyInner(epoch, minimumEntropy) - if err != nil { - return "", errors.Wrap(err, "url-key generation failed") - } - - found := pastesBucket.Get([]byte(key)) - if found == nil { - break - } - - isReserved := false - for _, reservedKey := range ReservedPasteKeys { - if strings.HasPrefix(key, reservedKey) { - isReserved = true - break - } - } - if !isReserved { - break - } - - epoch++ - } - return key, nil -} - -// generatePasteKeyInner generates a new paste key, but leaves the -// uniqueness and is-reserved checks to the caller. That is, it only -// generates a random key in the correct (syntactical) format. -// Both epoch and entropy can be used to set the key length. Epoch is used -// to prevent collisions in retrying to generate new keys. Entropy (in bits) -// is used to ensure that a new key has at least some amount of guessing -// entropy. -func generatePasteKeyInner(epoch, entropy int) (string, error) { - entropyEpoch := entropy - entropyEpoch -= minKeyLen * 6 // First 4 characters provide 24 bits. - entropyEpoch++ // One bit less because of '0' bit. - entropyEpoch = (entropyEpoch-1)/5 + 1 // 5 bits for every added epoch. - if epoch < entropyEpoch { - epoch = entropyEpoch - } - urlKey := make([]byte, minKeyLen+epoch) - _, err := rand.Read(urlKey) - if err != nil { - return "", err - } - // Put all the values in the range 0..64 for easier base64-encoding - for i := 0; i < len(urlKey); i++ { - urlKey[i] &= 0x3F - } - // Implement truncate-resistance by forcing the prefix to - // 0b111110xxxxxxxxxx - // ^----- {epoch} ones followed by a single 0 - // - // Example when epoch is 1: prefix is 0b10. - i := 0 - for i < epoch { - // Set this bit to 1 - limb := i / 6 - bit := i % 6 - urlKey[limb] |= 1 << uint(5-bit) - i++ - } - // Finally set the next bit to 0 - limb := i / 6 - bit := i % 6 - urlKey[limb] &= ^(1 << uint(5-bit)) - - // Convert this ID to a canonical base64 notation - for i := range urlKey { - urlKey[i] = base64Alphabet[urlKey[i]] - } - return string(urlKey), nil -} - -// GenerateDeleteToken generates a new (random) delete token. -func GenerateDeleteToken() (string, error) { - var deleteToken [16]byte - _, err := rand.Read(deleteToken[:]) - if err != nil { - return "", err - } - return hex.EncodeToString(deleteToken[:]), nil -} - -// AllPastes tries to retrieve all the Paste objects from the database. -func AllPastes(tx *bolt.Tx) ([]Paste, error) { - bucket := tx.Bucket([]byte(BucketPastes)) - if bucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketPastes) - } - var ps []Paste - err := bucket.ForEach(func(_, storedBytes []byte) error { - p, err := decodePaste(storedBytes) - if err != nil { - return err - } - ps = append(ps, *p) - return nil - }) - if err != nil { - return nil, err - } - return ps, nil -} diff --git a/internal/boltdb/paste_test.go b/internal/boltdb/paste_test.go deleted file mode 100644 index bba0d33..0000000 --- a/internal/boltdb/paste_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package boltdb - -import "testing" - -func TestValidatePasteKey(t *testing.T) { - tests := []struct { - key string - wantErr bool - errKind error - }{ - {"xd42__", false, nil}, - {"xd42_*", true, ErrKeyInvalidChar}, - {"xd42_/", true, ErrKeyInvalidChar}, - {"xd42_=", true, ErrKeyInvalidChar}, - {"xd42_", true, ErrKeyInvalidLength}, - {"xd42", true, ErrKeyInvalidLength}, - {"xd4", true, ErrKeyInvalidLength}, - {"xd", true, ErrKeyInvalidLength}, - {"x", true, ErrKeyInvalidLength}, - {"", true, ErrKeyInvalidLength}, - - {"KoJ5", false, nil}, - - {"__dGSJIIbBpr-SD0", false, nil}, - {"__dGSJIIbBpr-SD", true, ErrKeyInvalidLength}, - } - for _, tt := range tests { - t.Run(tt.key, func(t *testing.T) { - err := ValidatePasteKey(tt.key) - if (err != nil) != tt.wantErr { - t.Errorf("ValidatePasteKey() got error = %v, want error %v", err != nil, tt.wantErr) - } - if (err != nil) && err != tt.errKind { - t.Errorf("ValidatePasteKey() error = %v, want errKind %v", err, tt.errKind) - } - }) - } -}