Daan Sprenkels
3 years ago
parent
8b87cd0f8a
commit
0cfad96b68
@ -1,135 +0,0 @@
|
||||
package rushlink
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
var DB *bolt.DB
|
||||
|
||||
// The current database version
|
||||
//
|
||||
// If we alter the database format, we bump this number and write a new
|
||||
// database migration in migrateDatabase().
|
||||
const CURRENT_MIGRATE_VERSION = 2
|
||||
|
||||
// Bucket storing everything that is not a bulk value. This includes stuff like
|
||||
// the database version, secret site-wide keys.
|
||||
const BUCKET_CONF = "conf"
|
||||
|
||||
// The main bucket for paste values and URL redirects
|
||||
const BUCKET_PASTES = "pastes"
|
||||
|
||||
// The main bucket for file uploads
|
||||
const BUCKET_FILE_UPLOAD = "fileUpload"
|
||||
|
||||
// This value stores the current migration version. If this value is less than
|
||||
// CURRENT_MIGRATE_VERSION, the database has to be migrated.
|
||||
const KEY_MIGRATE_VERSION = "migrate_version"
|
||||
|
||||
// Open the bolt database
|
||||
func OpenDB(path string) error {
|
||||
if path == "" {
|
||||
return errors.New("database not set")
|
||||
}
|
||||
|
||||
var err error
|
||||
DB, err = bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to open database at '%v'", path)
|
||||
}
|
||||
return DB.Update(migrateDatabase)
|
||||
}
|
||||
|
||||
// Close the bolt database
|
||||
func CloseDB() error {
|
||||
if DB == nil {
|
||||
panic("no open database")
|
||||
}
|
||||
return DB.Close()
|
||||
}
|
||||
|
||||
// Initialize and migrate the database to the current version
|
||||
func migrateDatabase(tx *bolt.Tx) error {
|
||||
dbVersion, err := dbVersion(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrate the database to version 1
|
||||
if dbVersion < 1 {
|
||||
log.Println("migrating database to version 1")
|
||||
// Create conf bucket
|
||||
_, err := tx.CreateBucket([]byte(BUCKET_CONF))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Create paste bucket
|
||||
_, err = tx.CreateBucket([]byte(BUCKET_PASTES))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if dbVersion < 2 {
|
||||
log.Println("migrating database to version 2")
|
||||
// Create fileUpload bucket
|
||||
_, err := tx.CreateBucket([]byte(BUCKET_FILE_UPLOAD))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the current migrate version from the database
|
||||
func dbVersion(tx *bolt.Tx) (int, error) {
|
||||
conf := tx.Bucket([]byte(BUCKET_CONF))
|
||||
if conf == nil {
|
||||
return 0, nil
|
||||
}
|
||||
dbVersionBytes := conf.Get([]byte(KEY_MIGRATE_VERSION))
|
||||
if dbVersionBytes == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Version was already stored
|
||||
var dbVersion int
|
||||
if err := Unmarshal(dbVersionBytes, &dbVersion); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if dbVersion == 0 {
|
||||
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
|
||||
}
|
||||
if dbVersion > CURRENT_MIGRATE_VERSION {
|
||||
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CURRENT_MIGRATE_VERSION)
|
||||
}
|
||||
return dbVersion, nil
|
||||
}
|
||||
|
||||
// Update the current migrate version in the database
|
||||
func setDBVersion(tx *bolt.Tx, version int) error {
|
||||
conf, err := tx.CreateBucketIfNotExists([]byte(BUCKET_CONF))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionBytes, err := Marshal(version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conf.Put([]byte(KEY_MIGRATE_VERSION), versionBytes)
|
||||
}
|
||||
@ -0,0 +1,147 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
)
|
||||
|
||||
// Database is the main rushlink database type.
|
||||
//
|
||||
// Open a database using DB.Open() and close it in the end using DB.Close().
|
||||
// Only one instance of DB should exist in a program at any moment.
|
||||
type Database struct {
|
||||
Bolt *bolt.DB
|
||||
}
|
||||
|
||||
// CurrentMigrateVersion holds the current "migrate version".
|
||||
//
|
||||
// If we alter the database format, we bump this number and write a new
|
||||
// database migration in migrate().
|
||||
const CurrentMigrateVersion = 2
|
||||
|
||||
// BucketConf holds the name for the "configuration" bucket.
|
||||
//
|
||||
// This bucket holds the database version, secret site-wide keys, etc.
|
||||
const BucketConf = "conf"
|
||||
|
||||
// BucketPastes holds the name for the pastes bucket.
|
||||
const BucketPastes = "pastes"
|
||||
|
||||
// BucketFileUpload holds the name for the file-upload bucket.
|
||||
const BucketFileUpload = "fileUpload"
|
||||
|
||||
// KeyMigrateVersion stores the current migration version. If this value is less than
|
||||
// CurrentMigrateVersion, the database has to be migrated.
|
||||
const KeyMigrateVersion = "migrate_version"
|
||||
|
||||
// OpenDB opens a database file located at path.
|
||||
func OpenDB(path string) (*Database, error) {
|
||||
if path == "" {
|
||||
return nil, errors.New("database not set")
|
||||
}
|
||||
|
||||
db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to open database at '%v'", path)
|
||||
}
|
||||
if err := db.Update(migrate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Database{db}, nil
|
||||
}
|
||||
|
||||
// Close the bolt database
|
||||
func (db *Database) Close() error {
|
||||
if db == nil {
|
||||
panic("no open database")
|
||||
}
|
||||
return db.Close()
|
||||
}
|
||||
|
||||
// Initialize and migrate the database to the current version
|
||||
func migrate(tx *bolt.Tx) error {
|
||||
dbVersion, err := dbVersion(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrate the database to version 1
|
||||
if dbVersion < 1 {
|
||||
log.Println("migrating database to version 1")
|
||||
// Create conf bucket
|
||||
_, err := tx.CreateBucket([]byte(BucketConf))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Create paste bucket
|
||||
_, err = tx.CreateBucket([]byte(BucketPastes))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if dbVersion < 2 {
|
||||
log.Println("migrating database to version 2")
|
||||
// Create fileUpload bucket
|
||||
_, err := tx.CreateBucket([]byte(BucketFileUpload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 2); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the current migrate version from the database
|
||||
func dbVersion(tx *bolt.Tx) (int, error) {
|
||||
conf := tx.Bucket([]byte(BucketConf))
|
||||
if conf == nil {
|
||||
return 0, nil
|
||||
}
|
||||
dbVersionBytes := conf.Get([]byte(KeyMigrateVersion))
|
||||
if dbVersionBytes == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Version was already stored
|
||||
var dbVersion int
|
||||
if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if dbVersion == 0 {
|
||||
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
|
||||
}
|
||||
if dbVersion > CurrentMigrateVersion {
|
||||
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion)
|
||||
}
|
||||
return dbVersion, nil
|
||||
}
|
||||
|
||||
// Update the current migrate version in the database
|
||||
func setDBVersion(tx *bolt.Tx, version int) error {
|
||||
conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionBytes, err := gobmarsh.Marshal(version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conf.Put([]byte(KeyMigrateVersion), versionBytes)
|
||||
}
|
||||
@ -0,0 +1,184 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
)
|
||||
|
||||
// Use the Castagnoli checksum because of the acceleration on Intel CPUs
|
||||
var checksumTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// FileStore holds the path to a file storage location.
|
||||
type FileStore struct {
|
||||
path string
|
||||
}
|
||||
|
||||
// FileUploadState determines the current state of a FileUpload object.
|
||||
type FileUploadState int
|
||||
|
||||
// FileUpload models an uploaded file.
|
||||
type FileUpload struct {
|
||||
State FileUploadState
|
||||
ID uuid.UUID
|
||||
FileName string
|
||||
ContentType string
|
||||
Checksum uint32
|
||||
}
|
||||
|
||||
const (
|
||||
dirMode os.FileMode = 0750
|
||||
fileMode os.FileMode = 0640
|
||||
)
|
||||
|
||||
const (
|
||||
// FileUploadStateUndef is an undefined FileUpload.
|
||||
FileUploadStateUndef FileUploadState = 0
|
||||
// FileUploadStatePresent denotes the normal (existing) state.
|
||||
FileUploadStatePresent FileUploadState = 1
|
||||
// FileUploadStateDeleted denotes a deleted state.
|
||||
FileUploadStateDeleted FileUploadState = 2
|
||||
)
|
||||
|
||||
func (t FileUploadState) String() string {
|
||||
switch t {
|
||||
case FileUploadStateUndef:
|
||||
return "unknown"
|
||||
case FileUploadStatePresent:
|
||||
return "present"
|
||||
case FileUploadStateDeleted:
|
||||
return "deleted"
|
||||
default:
|
||||
return "invalid"
|
||||
}
|
||||
}
|
||||
|
||||
// OpenFileStore opens the file storage at path.
|
||||
func OpenFileStore(path string) (*FileStore, error) {
|
||||
if path == "" {
|
||||
return nil, errors.New("file-store not set")
|
||||
}
|
||||
|
||||
// Try to create the file store directory if it does not yet exist
|
||||
if err := os.MkdirAll(path, dirMode); err != nil {
|
||||
return nil, errors.Wrap(err, "creating file store directory")
|
||||
}
|
||||
|
||||
return &FileStore{path[:]}, nil
|
||||
}
|
||||
|
||||
// NewFileUpload creates a new FileUpload object.
|
||||
func NewFileUpload(fs *FileStore, r io.Reader, fileName string, contentType string) (*FileUpload, error) {
|
||||
id, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "generating UUID")
|
||||
}
|
||||
|
||||
filePath := fs.FilePath(id, fileName)
|
||||
if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil {
|
||||
return nil, errors.Wrap(err, "creating file dir")
|
||||
}
|
||||
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "opening file")
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := crc32.New(checksumTable)
|
||||
tee := io.TeeReader(r, hash)
|
||||
_, err = io.Copy(file, tee)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "writing to file")
|
||||
}
|
||||
|
||||
fu := &FileUpload{
|
||||
State: FileUploadStatePresent,
|
||||
ID: id,
|
||||
FileName: fileName,
|
||||
ContentType: contentType,
|
||||
Checksum: hash.Sum32(),
|
||||
}
|
||||
return fu, nil
|
||||
}
|
||||
|
||||
func (fs *FileStore) Path() string {
|
||||
return fs.path
|
||||
}
|
||||
|
||||
func (fs *FileStore) FilePath(id uuid.UUID, fileName string) string {
|
||||
if fs.path == "" {
|
||||
panic("fileStoreDir called while the file store path has not been set")
|
||||
}
|
||||
return path.Join(fs.path, hex.EncodeToString(id[:]), fileName)
|
||||
}
|
||||
|
||||
func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) {
|
||||
bucket := tx.Bucket([]byte(BucketFileUpload))
|
||||
if bucket == nil {
|
||||
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
|
||||
}
|
||||
storedBytes := bucket.Get(id[:])
|
||||
if storedBytes == nil {
|
||||
return nil, nil
|
||||
}
|
||||
fu := &FileUpload{}
|
||||
err := gobmarsh.Unmarshal(storedBytes, fu)
|
||||
return fu, err
|
||||
}
|
||||
|
||||
// Save saves a FileUpload in the database.
|
||||
func (fu *FileUpload) Save(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(BucketFileUpload))
|
||||
if bucket == nil {
|
||||
return errors.Errorf("bucket %v does not exist", BucketFileUpload)
|
||||
}
|
||||
|
||||
buf, err := gobmarsh.Marshal(fu)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "encoding for database failed")
|
||||
}
|
||||
if err := bucket.Put(fu.ID[:], buf); err != nil {
|
||||
return errors.Wrap(err, "database transaction failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes a FileUpload from the database.
|
||||
func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
|
||||
// Remove the file in the backend
|
||||
filePath := fs.FilePath(fu.ID, fu.FileName)
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update the file in the server
|
||||
if err := (&FileUpload{
|
||||
ID: fu.ID,
|
||||
State: FileUploadStateDeleted,
|
||||
}).Save(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Cleanup the parent directory
|
||||
wrap := "deletion succeeded, but removing the file directory has failed"
|
||||
return errors.Wrap(os.Remove(path.Dir(filePath)), wrap)
|
||||
}
|
||||
|
||||
// URL returns the URL for the FileUpload.
|
||||
func (fu *FileUpload) URL() *url.URL {
|
||||
rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName
|
||||
urlParse, err := url.Parse(rawurl)
|
||||
if err != nil {
|
||||
panic("could not construct /uploads/ url")
|
||||
}
|
||||
return urlParse
|
||||
}
|
||||
Loading…
Reference in new issue