Remove boltdb leftovers
This commit is contained in:
Daan Sprenkels
parent
38b27b4d11
commit
9ff11cc14c
@ -1,110 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitea.hashru.nl/dsprenkels/rushlink/internal/boltdb"
|
||||
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var (
|
||||
boltDBPath = flag.String("boltdb", "", "location of the bolt database file")
|
||||
fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
boltFileStore, err := boltdb.OpenFileStore(*fileStorePath)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
boltDB, err := boltdb.OpenDB(*boltDBPath, boltFileStore)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
defer boltDB.Close()
|
||||
sqlDB, err := db.OpenDBFromEnvironment()
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// Migrate database schema
|
||||
m := db.Gormigrate(sqlDB)
|
||||
if err := m.MigrateTo("202010251337"); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
// Migrate all files in filestorage
|
||||
if err := sqlDB.Transaction(func(sqlTx *gorm.DB) error {
|
||||
return boltDB.Bolt.View(func(boltTx *bolt.Tx) error {
|
||||
// Migrate all the file uploads
|
||||
allFUs, err := boltdb.AllFileUploads(boltTx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var fusDeleted uint
|
||||
var sqlFUs []db.FileUpload
|
||||
for _, fu := range allFUs {
|
||||
isDeleted := fu.State == boltdb.FileUploadStateDeleted
|
||||
sqlFU := db.FileUpload{
|
||||
State: db.FileUploadState(int(fu.State)),
|
||||
PubID: fu.ID,
|
||||
FileName: fu.FileName,
|
||||
ContentType: fu.ContentType,
|
||||
Checksum: fu.Checksum,
|
||||
DeletedAt: deletedAt(isDeleted),
|
||||
}
|
||||
sqlFUs = append(sqlFUs, sqlFU)
|
||||
if isDeleted {
|
||||
fusDeleted++
|
||||
}
|
||||
}
|
||||
log.Printf("migrating %v file uploads (of which %v deleted)", len(sqlFUs), fusDeleted)
|
||||
sqlTx.Create(sqlFUs)
|
||||
|
||||
// Migrate all the pastes.
|
||||
allPastes, err := boltdb.AllPastes(boltTx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pastesDeleted uint
|
||||
var sqlPastes []db.Paste
|
||||
for _, paste := range allPastes {
|
||||
isDeleted := paste.State == boltdb.PasteStateDeleted
|
||||
sqlPaste := db.Paste{
|
||||
Type: db.PasteType(int(paste.Type)),
|
||||
State: db.PasteState(int(paste.State)),
|
||||
Content: paste.Content,
|
||||
Key: paste.Key,
|
||||
DeleteToken: paste.DeleteToken,
|
||||
CreatedAt: paste.TimeCreated,
|
||||
DeletedAt: deletedAt(isDeleted),
|
||||
}
|
||||
sqlPastes = append(sqlPastes, sqlPaste)
|
||||
if isDeleted {
|
||||
pastesDeleted++
|
||||
}
|
||||
}
|
||||
log.Printf("migrating %v pastes (of which %v deleted)", len(sqlPastes), pastesDeleted)
|
||||
sqlTx.Create(sqlPastes)
|
||||
|
||||
return nil
|
||||
})
|
||||
}); err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
|
||||
log.Println("migration successful! :D")
|
||||
}
|
||||
|
||||
func deletedAt(isDeleted bool) gorm.DeletedAt {
|
||||
if isDeleted {
|
||||
return gorm.DeletedAt{Time: time.Now(), Valid: true}
|
||||
}
|
||||
return gorm.DeletedAt{}
|
||||
}
|
||||
1
go.mod
1
go.mod
@ -10,7 +10,6 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.10.0
|
||||
github.com/prometheus/common v0.23.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.5
|
||||
golang.org/x/sys v0.0.0-20210507161434-a76c4d0a0096 // indirect
|
||||
gorm.io/driver/postgres v1.1.0
|
||||
gorm.io/driver/sqlite v1.1.4
|
||||
|
||||
3
go.sum
3
go.sum
@ -382,8 +382,6 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
|
||||
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
@ -471,7 +469,6 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
||||
@ -1,204 +0,0 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
)
|
||||
|
||||
// Database is the main rushlink database type.
|
||||
//
|
||||
// Open a database using DB.Open() and close it in the end using DB.Close().
|
||||
// Only one instance of DB should exist in a program at any moment.
|
||||
type Database struct {
|
||||
Bolt *bolt.DB
|
||||
}
|
||||
|
||||
// CurrentMigrateVersion holds the current "migrate version".
|
||||
//
|
||||
// If we alter the database format, we bump this number and write a new
|
||||
// database migration in migrate().
|
||||
const CurrentMigrateVersion = 3
|
||||
|
||||
// BucketConf holds the name for the "configuration" bucket.
|
||||
//
|
||||
// This bucket holds the database version, secret site-wide keys, etc.
|
||||
const BucketConf = "conf"
|
||||
|
||||
// BucketPastes holds the name for the pastes bucket.
|
||||
const BucketPastes = "pastes"
|
||||
|
||||
// BucketFileUpload holds the name for the file-upload bucket.
|
||||
const BucketFileUpload = "fileUpload"
|
||||
|
||||
// KeyMigrateVersion stores the current migration version. If this value is less than
|
||||
// CurrentMigrateVersion, the database has to be migrated.
|
||||
const KeyMigrateVersion = "migrate_version"
|
||||
|
||||
// OpenDB opens a database file located at path.
|
||||
func OpenDB(path string, fs *FileStore) (*Database, error) {
|
||||
if path == "" {
|
||||
return nil, errors.New("database not set")
|
||||
}
|
||||
|
||||
db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to open database at '%v'", path)
|
||||
}
|
||||
if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Database{db}, nil
|
||||
}
|
||||
|
||||
// Close the bolt database
|
||||
func (db *Database) Close() error {
|
||||
if db == nil {
|
||||
panic("no open database")
|
||||
}
|
||||
return db.Bolt.Close()
|
||||
}
|
||||
|
||||
// Initialize and migrate the database to the current version
|
||||
func migrate(tx *bolt.Tx, fs *FileStore) error {
|
||||
// Guidelines for error handling:
|
||||
// - Errors based on malformed *structure* should be fatal!
|
||||
// - Errors based on malformed *data* should print a warning
|
||||
// (and if possible try to fix the error).
|
||||
|
||||
dbVersion, err := dbVersion(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrate the database to version 1
|
||||
if dbVersion < 1 {
|
||||
log.Println("migrating database to version 1")
|
||||
// Create conf bucket
|
||||
_, err := tx.CreateBucket([]byte(BucketConf))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Create paste bucket
|
||||
_, err = tx.CreateBucket([]byte(BucketPastes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if dbVersion < 2 {
|
||||
log.Println("migrating database to version 2")
|
||||
// Create fileUpload bucket
|
||||
_, err := tx.CreateBucket([]byte(BucketFileUpload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if dbVersion < 3 {
|
||||
log.Println("migrating database to version 3")
|
||||
// In this version, we changed te way how Content-Types are being
|
||||
// stored. Previously, we allowed clients to provide their own
|
||||
// Content-Types for files, using the Content-Disposition header in
|
||||
// multipart forms. The new way detects these types using
|
||||
// http.DetectContentType.
|
||||
//
|
||||
// Scan through all the FileUploads and update their ContentTypes.
|
||||
bucket := tx.Bucket([]byte(BucketFileUpload))
|
||||
cursor := bucket.Cursor()
|
||||
var id, storedBytes []byte
|
||||
id, storedBytes = cursor.First()
|
||||
for id != nil {
|
||||
fu, err := decodeFileUpload(storedBytes)
|
||||
if err != nil {
|
||||
log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id))
|
||||
id, storedBytes = cursor.Next()
|
||||
continue
|
||||
}
|
||||
if fu.State != FileUploadStatePresent {
|
||||
id, storedBytes = cursor.Next()
|
||||
continue
|
||||
}
|
||||
filePath := fu.Path(fs)
|
||||
file, err := os.Open(fu.Path(fs))
|
||||
if err != nil {
|
||||
log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath))
|
||||
id, storedBytes = cursor.Next()
|
||||
continue
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
buf.Grow(512)
|
||||
io.CopyN(&buf, file, 512)
|
||||
contentType := http.DetectContentType(buf.Bytes())
|
||||
if contentType != fu.ContentType {
|
||||
fu.ContentType = contentType
|
||||
fu.Save(tx)
|
||||
cursor.Seek(id)
|
||||
}
|
||||
id, storedBytes = cursor.Next()
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 3); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the current migrate version from the database
|
||||
func dbVersion(tx *bolt.Tx) (int, error) {
|
||||
conf := tx.Bucket([]byte(BucketConf))
|
||||
if conf == nil {
|
||||
return 0, nil
|
||||
}
|
||||
dbVersionBytes := conf.Get([]byte(KeyMigrateVersion))
|
||||
if dbVersionBytes == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Version was already stored
|
||||
var dbVersion int
|
||||
if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if dbVersion == 0 {
|
||||
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
|
||||
}
|
||||
if dbVersion > CurrentMigrateVersion {
|
||||
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion)
|
||||
}
|
||||
return dbVersion, nil
|
||||
}
|
||||
|
||||
// Update the current migrate version in the database
|
||||
func setDBVersion(tx *bolt.Tx, version int) error {
|
||||
conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionBytes, err := gobmarsh.Marshal(version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conf.Put([]byte(KeyMigrateVersion), versionBytes)
|
||||
}
|
||||
@ -1,256 +0,0 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
)
|
||||
|
||||
// Use the Castagnoli checksum because of the acceleration on Intel CPUs
|
||||
var checksumTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// FileStore holds the path to a file storage location.
|
||||
type FileStore struct {
|
||||
path string
|
||||
}
|
||||
|
||||
// FileUploadState determines the current state of a FileUpload object.
|
||||
type FileUploadState int
|
||||
|
||||
// FileUpload models an uploaded file.
|
||||
type FileUpload struct {
|
||||
// State of the FileUpload (present/deleted/etc).
|
||||
State FileUploadState
|
||||
|
||||
// ID identifies this FileUpload.
|
||||
ID uuid.UUID
|
||||
|
||||
// FileName contains the original filename of this FileUpload.
|
||||
FileName string
|
||||
|
||||
// Content type as determined by http.DetectContentType.
|
||||
ContentType string
|
||||
|
||||
// Checksum holds a crc32c checksum of the file.
|
||||
//
|
||||
// This checksum is only meant to allow for the detection of random
|
||||
// database corruption.
|
||||
Checksum uint32
|
||||
}
|
||||
|
||||
const (
|
||||
dirMode os.FileMode = 0750
|
||||
fileMode os.FileMode = 0640
|
||||
)
|
||||
|
||||
const (
|
||||
// FileUploadStateUndef is an undefined FileUpload.
|
||||
FileUploadStateUndef FileUploadState = 0
|
||||
// FileUploadStatePresent denotes the normal (existing) state.
|
||||
FileUploadStatePresent FileUploadState = 1
|
||||
// FileUploadStateDeleted denotes a deleted state.
|
||||
FileUploadStateDeleted FileUploadState = 2
|
||||
)
|
||||
|
||||
func (t FileUploadState) String() string {
|
||||
switch t {
|
||||
case FileUploadStateUndef:
|
||||
return "unknown"
|
||||
case FileUploadStatePresent:
|
||||
return "present"
|
||||
case FileUploadStateDeleted:
|
||||
return "deleted"
|
||||
default:
|
||||
return "invalid"
|
||||
}
|
||||
}
|
||||
|
||||
// OpenFileStore opens the file storage at path.
|
||||
func OpenFileStore(path string) (*FileStore, error) {
|
||||
if path == "" {
|
||||
return nil, errors.New("file-store not set")
|
||||
}
|
||||
|
||||
// Try to create the file store directory if it does not yet exist
|
||||
if err := os.MkdirAll(path, dirMode); err != nil {
|
||||
return nil, errors.Wrap(err, "creating file store directory")
|
||||
}
|
||||
|
||||
return &FileStore{path[:]}, nil
|
||||
}
|
||||
|
||||
// Path returns the path of the FileStore root.
|
||||
func (fs *FileStore) Path() string {
|
||||
return fs.path
|
||||
}
|
||||
|
||||
// filePath resolves the path of a file in the FileStore given some id and filename.
|
||||
func (fs *FileStore) filePath(id uuid.UUID, fileName string) string {
|
||||
if fs.path == "" {
|
||||
panic("fileStoreDir called while the file store path has not been set")
|
||||
}
|
||||
return path.Join(fs.path, hex.EncodeToString(id[:]), fileName)
|
||||
}
|
||||
|
||||
// NewFileUpload creates a new FileUpload object.
|
||||
//
|
||||
// Internally, this function detects the type of the file stored in `r` using
|
||||
// `http.DetectContentType`.
|
||||
func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) {
|
||||
// Generate a file ID
|
||||
id, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "generating UUID")
|
||||
}
|
||||
|
||||
// Construct a checksum for this file
|
||||
hash := crc32.New(checksumTable)
|
||||
tee := io.TeeReader(r, hash)
|
||||
|
||||
// Detect the file type
|
||||
var tmpBuf bytes.Buffer
|
||||
tmpBuf.Grow(512)
|
||||
io.CopyN(&tmpBuf, tee, 512)
|
||||
contentType := http.DetectContentType(tmpBuf.Bytes())
|
||||
|
||||
// Open the file on disk for writing
|
||||
baseName := filepath.Base(fileName)
|
||||
filePath := fs.filePath(id, baseName)
|
||||
if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil {
|
||||
return nil, errors.Wrap(err, "creating file dir")
|
||||
}
|
||||
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "opening file")
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Write the file to disk
|
||||
_, err = io.Copy(file, &tmpBuf)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "writing to file")
|
||||
}
|
||||
_, err = io.Copy(file, tee)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "writing to file")
|
||||
}
|
||||
|
||||
fu := &FileUpload{
|
||||
State: FileUploadStatePresent,
|
||||
ID: id,
|
||||
FileName: baseName,
|
||||
ContentType: contentType,
|
||||
Checksum: hash.Sum32(),
|
||||
}
|
||||
return fu, nil
|
||||
}
|
||||
|
||||
// GetFileUpload tries to retrieve a FileUpload object from the bolt database.
|
||||
func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) {
|
||||
bucket := tx.Bucket([]byte(BucketFileUpload))
|
||||
if bucket == nil {
|
||||
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
|
||||
}
|
||||
storedBytes := bucket.Get(id[:])
|
||||
if storedBytes == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return decodeFileUpload(storedBytes)
|
||||
}
|
||||
|
||||
// AllFileUploads tries to retrieve all FileUpload objects from the bolt database.
|
||||
func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) {
|
||||
bucket := tx.Bucket([]byte(BucketFileUpload))
|
||||
if bucket == nil {
|
||||
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
|
||||
}
|
||||
var fus []FileUpload
|
||||
err := bucket.ForEach(func(_, storedBytes []byte) error {
|
||||
fu, err := decodeFileUpload(storedBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fus = append(fus, *fu)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fus, nil
|
||||
}
|
||||
|
||||
func decodeFileUpload(storedBytes []byte) (*FileUpload, error) {
|
||||
fu := &FileUpload{}
|
||||
err := gobmarsh.Unmarshal(storedBytes, fu)
|
||||
return fu, err
|
||||
}
|
||||
|
||||
// Save saves a FileUpload in the database.
|
||||
func (fu *FileUpload) Save(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(BucketFileUpload))
|
||||
if bucket == nil {
|
||||
return errors.Errorf("bucket %v does not exist", BucketFileUpload)
|
||||
}
|
||||
|
||||
buf, err := gobmarsh.Marshal(fu)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "encoding for database failed")
|
||||
}
|
||||
if err := bucket.Put(fu.ID[:], buf); err != nil {
|
||||
return errors.Wrap(err, "database transaction failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes a FileUpload from the database.
|
||||
func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
|
||||
// Remove the file in the backend
|
||||
filePath := fu.Path(fs)
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update the file in the server
|
||||
if err := (&FileUpload{
|
||||
ID: fu.ID,
|
||||
State: FileUploadStateDeleted,
|
||||
}).Save(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Cleanup the parent directory
|
||||
wrap := "deletion succeeded, but removing the file directory has failed"
|
||||
return errors.Wrap(os.Remove(path.Dir(filePath)), wrap)
|
||||
}
|
||||
|
||||
// Path returns the path to this FileUpload in the FileStore provided in fs.
|
||||
func (fu *FileUpload) Path(fs *FileStore) string {
|
||||
return fs.filePath(fu.ID, fu.FileName)
|
||||
}
|
||||
|
||||
// URL returns the URL for the FileUpload.
|
||||
func (fu *FileUpload) URL() *url.URL {
|
||||
rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName
|
||||
urlParse, err := url.Parse(rawurl)
|
||||
if err != nil {
|
||||
panic("could not construct /uploads/ url")
|
||||
}
|
||||
return urlParse
|
||||
}
|
||||
|
||||
// Ext returns the extension of the file attached to this FileUpload.
|
||||
func (fu *FileUpload) Ext() string {
|
||||
return filepath.Ext(fu.FileName)
|
||||
}
|
||||
@ -1,345 +0,0 @@
|
||||
package boltdb
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
// PasteType describes the type of Paste (i.e. file, redirect, [...]).
|
||||
type PasteType int
|
||||
|
||||
// PasteState describes the state of a Paste (i.e. present, deleted, [...]).
|
||||
type PasteState int
|
||||
|
||||
// Paste describes the main Paste model in the database.
|
||||
type Paste struct {
|
||||
Type PasteType
|
||||
State PasteState
|
||||
Content []byte
|
||||
Key string
|
||||
DeleteToken string
|
||||
TimeCreated time.Time
|
||||
}
|
||||
|
||||
// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen
|
||||
var ReservedPasteKeys = []string{"xd42", "example"}
|
||||
|
||||
// Note: we use iota here. That means removals of PasteType* are not allowed,
|
||||
// because this changes the value of the constant. Please add the comment
|
||||
// "// deprecated" if you want to remove the constant. Additions are only
|
||||
// allowed at the bottom of this block, for the same reason.
|
||||
const (
|
||||
PasteTypeUndef PasteType = iota
|
||||
// PasteTypePaste is as of yet unused. It is still unclear if this type
|
||||
// will ever get a proper meaning.
|
||||
PasteTypePaste
|
||||
PasteTypeRedirect
|
||||
PasteTypeFileUpload
|
||||
)
|
||||
|
||||
// Note: we use iota here. See the comment above PasteType*
|
||||
const (
|
||||
PasteStateUndef PasteState = iota
|
||||
PasteStatePresent
|
||||
PasteStateDeleted
|
||||
)
|
||||
|
||||
// minKeyLen specifies the mimimum length of a paste key.
|
||||
const minKeyLen = 4
|
||||
|
||||
var (
|
||||
// ErrKeyInvalidChar occurs when a key contains an invalid character.
|
||||
ErrKeyInvalidChar = errors.New("invalid character in key")
|
||||
// ErrKeyInvalidLength occurs when a key embeds a length that is incorrect.
|
||||
ErrKeyInvalidLength = errors.New("key length encoding is incorrect")
|
||||
// ErrPasteDoesNotExist occurs when a key does not exist in the database.
|
||||
ErrPasteDoesNotExist = errors.New("url key not found in the database")
|
||||
)
|
||||
|
||||
// ErrHTTPStatusCode returns the HTTP status code that should correspond to
|
||||
// the provided error.
|
||||
// server error, or false if it is not.
|
||||
func ErrHTTPStatusCode(err error) int {
|
||||
switch err {
|
||||
case nil:
|
||||
return 0
|
||||
case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist:
|
||||
return http.StatusNotFound
|
||||
}
|
||||
return http.StatusInternalServerError
|
||||
}
|
||||
|
||||
// Base64 encoding and decoding
|
||||
var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
|
||||
var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding)
|
||||
|
||||
func (t PasteType) String() string {
|
||||
switch t {
|
||||
case PasteTypeUndef:
|
||||
return "unknown"
|
||||
case PasteTypePaste:
|
||||
return "paste"
|
||||
case PasteTypeRedirect:
|
||||
return "redirect"
|
||||
case PasteTypeFileUpload:
|
||||
return "file"
|
||||
default:
|
||||
return "invalid"
|
||||
}
|
||||
}
|
||||
|
||||
func (t PasteState) String() string {
|
||||
switch t {
|
||||
case PasteStateUndef:
|
||||
return "unknown"
|
||||
case PasteStatePresent:
|
||||
return "present"
|
||||
case PasteStateDeleted:
|
||||
return "deleted"
|
||||
default:
|
||||
return "invalid"
|
||||
}
|
||||
}
|
||||
|
||||
// GetPaste retrieves a paste from the database.
|
||||
func GetPaste(tx *bolt.Tx, key string) (*Paste, error) {
|
||||
if err := ValidatePasteKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return GetPasteNoValidate(tx, key)
|
||||
}
|
||||
|
||||
// ValidatePasteKey validates the format of the key that has
|
||||
func ValidatePasteKey(key string) error {
|
||||
internalLen := minKeyLen
|
||||
countingOnes := true
|
||||
for _, ch := range key {
|
||||
limb := strings.IndexRune(base64Alphabet, ch)
|
||||
if limb == -1 {
|
||||
return ErrKeyInvalidChar
|
||||
}
|
||||
for i := 5; i >= 0 && countingOnes; i-- {
|
||||
if (limb>>uint(i))&0x1 == 0 {
|
||||
countingOnes = false
|
||||
break
|
||||
}
|
||||
internalLen++
|
||||
}
|
||||
}
|
||||
if internalLen != len(key) {
|
||||
return ErrKeyInvalidLength
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetPasteNoValidate retrieves a paste from the database without validating
|
||||
// the key format first.
|
||||
func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) {
|
||||
pastesBucket := tx.Bucket([]byte(BucketPastes))
|
||||
if pastesBucket == nil {
|
||||
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
|
||||
}
|
||||
storedBytes := pastesBucket.Get([]byte(key))
|
||||
if storedBytes == nil {
|
||||
return nil, ErrPasteDoesNotExist
|
||||
}
|
||||
return decodePaste(storedBytes)
|
||||
}
|
||||
|
||||
func decodePaste(storedBytes []byte) (*Paste, error) {
|
||||
p := &Paste{}
|
||||
err := gobmarsh.Unmarshal(storedBytes, p)
|
||||
return p, err
|
||||
}
|
||||
|
||||
// Save saves this Paste to the database.
|
||||
func (p *Paste) Save(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(BucketPastes))
|
||||
if bucket == nil {
|
||||
return errors.Errorf("bucket %v does not exist", BucketPastes)
|
||||
}
|
||||
|
||||
buf, err := gobmarsh.Marshal(p)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "encoding for database failed")
|
||||
}
|
||||
if err := bucket.Put([]byte(p.Key), buf); err != nil {
|
||||
return errors.Wrap(err, "database transaction failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes this Paste from the database.
|
||||
func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error {
|
||||
// Remove the (maybe) attached file
|
||||
if p.Type == PasteTypeFileUpload {
|
||||
fuID, err := uuid.FromBytes(p.Content)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to parse uuid")
|
||||
}
|
||||
fu, err := GetFileUpload(tx, fuID)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to find file in database")
|
||||
}
|
||||
if err := fu.Delete(tx, fs); err != nil {
|
||||
return errors.Wrap(err, "failed to remove file")
|
||||
}
|
||||
}
|
||||
|
||||
// Replace the old paste with a new empty paste
|
||||
p.Type = PasteTypeUndef
|
||||
p.State = PasteStateDeleted
|
||||
p.Content = []byte{}
|
||||
if err := p.Save(tx); err != nil {
|
||||
return errors.Wrap(err, "failed to delete paste in database")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RedirectURL returns the URL from this paste.
|
||||
//
|
||||
// This function assumes that the paste is valid. If the paste struct is
|
||||
// corrupted in some way, this function will panic.
|
||||
func (p *Paste) RedirectURL() *url.URL {
|
||||
if p.Type != PasteTypeRedirect {
|
||||
panic("expected p.Type to be PasteTypeRedirect")
|
||||
}
|
||||
rawurl := string(p.Content)
|
||||
urlParse, err := url.Parse(rawurl)
|
||||
if err != nil {
|
||||
panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key))
|
||||
}
|
||||
return urlParse
|
||||
}
|
||||
|
||||
// GeneratePasteKey generates a new paste key. It will ensure that the newly
|
||||
// generated paste key does not already exist in the database.
|
||||
// The running time of this function is in O(log N), where N is the amount of
|
||||
// keys stored in the url-shorten database.
|
||||
// In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum
|
||||
// guessing entropy of the generated key.
|
||||
func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) {
|
||||
pastesBucket := tx.Bucket([]byte(BucketPastes))
|
||||
if pastesBucket == nil {
|
||||
return "", errors.Errorf("bucket %v does not exist", BucketPastes)
|
||||
}
|
||||
|
||||
epoch := 0
|
||||
var key string
|
||||
for {
|
||||
var err error
|
||||
key, err = generatePasteKeyInner(epoch, minimumEntropy)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "url-key generation failed")
|
||||
}
|
||||
|
||||
found := pastesBucket.Get([]byte(key))
|
||||
if found == nil {
|
||||
break
|
||||
}
|
||||
|
||||
isReserved := false
|
||||
for _, reservedKey := range ReservedPasteKeys {
|
||||
if strings.HasPrefix(key, reservedKey) {
|
||||
isReserved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !isReserved {
|
||||
break
|
||||
}
|
||||
|
||||
epoch++
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// generatePasteKeyInner generates a new paste key, but leaves the
|
||||
// uniqueness and is-reserved checks to the caller. That is, it only
|
||||
// generates a random key in the correct (syntactical) format.
|
||||
// Both epoch and entropy can be used to set the key length. Epoch is used
|
||||
// to prevent collisions in retrying to generate new keys. Entropy (in bits)
|
||||
// is used to ensure that a new key has at least some amount of guessing
|
||||
// entropy.
|
||||
func generatePasteKeyInner(epoch, entropy int) (string, error) {
|
||||
entropyEpoch := entropy
|
||||
entropyEpoch -= minKeyLen * 6 // First 4 characters provide 24 bits.
|
||||
entropyEpoch++ // One bit less because of '0' bit.
|
||||
entropyEpoch = (entropyEpoch-1)/5 + 1 // 5 bits for every added epoch.
|
||||
if epoch < entropyEpoch {
|
||||
epoch = entropyEpoch
|
||||
}
|
||||
urlKey := make([]byte, minKeyLen+epoch)
|
||||
_, err := rand.Read(urlKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Put all the values in the range 0..64 for easier base64-encoding
|
||||
for i := 0; i < len(urlKey); i++ {
|
||||
urlKey[i] &= 0x3F
|
||||
}
|
||||
// Implement truncate-resistance by forcing the prefix to
|
||||
// 0b111110xxxxxxxxxx
|
||||
// ^----- {epoch} ones followed by a single 0
|
||||
//
|
||||
// Example when epoch is 1: prefix is 0b10.
|
||||
i := 0
|
||||
for i < epoch {
|
||||
// Set this bit to 1
|
||||
limb := i / 6
|
||||
bit := i % 6
|
||||
urlKey[limb] |= 1 << uint(5-bit)
|
||||
i++
|
||||
}
|
||||
// Finally set the next bit to 0
|
||||
limb := i / 6
|
||||
bit := i % 6
|
||||
urlKey[limb] &= ^(1 << uint(5-bit))
|
||||
|
||||
// Convert this ID to a canonical base64 notation
|
||||
for i := range urlKey {
|
||||
urlKey[i] = base64Alphabet[urlKey[i]]
|
||||
}
|
||||
return string(urlKey), nil
|
||||
}
|
||||
|
||||
// GenerateDeleteToken generates a new (random) delete token.
|
||||
func GenerateDeleteToken() (string, error) {
|
||||
var deleteToken [16]byte
|
||||
_, err := rand.Read(deleteToken[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(deleteToken[:]), nil
|
||||
}
|
||||
|
||||
// AllPastes tries to retrieve all the Paste objects from the database.
|
||||
func AllPastes(tx *bolt.Tx) ([]Paste, error) {
|
||||
bucket := tx.Bucket([]byte(BucketPastes))
|
||||
if bucket == nil {
|
||||
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
|
||||
}
|
||||
var ps []Paste
|
||||
err := bucket.ForEach(func(_, storedBytes []byte) error {
|
||||
p, err := decodePaste(storedBytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ps = append(ps, *p)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ps, nil
|
||||
}
|
||||
@ -1,38 +0,0 @@
|
||||
package boltdb
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidatePasteKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
key string
|
||||
wantErr bool
|
||||
errKind error
|
||||
}{
|
||||
{"xd42__", false, nil},
|
||||
{"xd42_*", true, ErrKeyInvalidChar},
|
||||
{"xd42_/", true, ErrKeyInvalidChar},
|
||||
{"xd42_=", true, ErrKeyInvalidChar},
|
||||
{"xd42_", true, ErrKeyInvalidLength},
|
||||
{"xd42", true, ErrKeyInvalidLength},
|
||||
{"xd4", true, ErrKeyInvalidLength},
|
||||
{"xd", true, ErrKeyInvalidLength},
|
||||
{"x", true, ErrKeyInvalidLength},
|
||||
{"", true, ErrKeyInvalidLength},
|
||||
|
||||
{"KoJ5", false, nil},
|
||||
|
||||
{"__dGSJIIbBpr-SD0", false, nil},
|
||||
{"__dGSJIIbBpr-SD", true, ErrKeyInvalidLength},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key, func(t *testing.T) {
|
||||
err := ValidatePasteKey(tt.key)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidatePasteKey() got error = %v, want error %v", err != nil, tt.wantErr)
|
||||
}
|
||||
if (err != nil) && err != tt.errKind {
|
||||
t.Errorf("ValidatePasteKey() error = %v, want errKind %v", err, tt.errKind)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user