electricdusk/rushlink
3
3
Fork 3
Code Issues 21 Pull Requests 2 Releases Activity

Remove boltdb leftovers

Browse Source
This commit is contained in:
Daan Sprenkels 2021-05-13 10:44:32 +02:00
parent 38b27b4d11
commit 9ff11cc14c
7 changed files with 0 additions and 957 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
cmd/rushlink-migrate-db/main.go
View File

@ -1,110 +0,0 @@
package main
import (
"flag"
"log"
"time"
"gitea.hashru.nl/dsprenkels/rushlink/internal/boltdb"
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
bolt "go.etcd.io/bbolt"
"gorm.io/gorm"
)
var (
boltDBPath = flag.String("boltdb", "", "location of the bolt database file")
fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored")
)
func main() {
flag.Parse()
boltFileStore, err := boltdb.OpenFileStore(*fileStorePath)
if err != nil {
log.Fatalln(err)
}
boltDB, err := boltdb.OpenDB(*boltDBPath, boltFileStore)
if err != nil {
log.Fatalln(err)
}
defer boltDB.Close()
sqlDB, err := db.OpenDBFromEnvironment()
if err != nil {
log.Fatalln(err)
}
// Migrate database schema
m := db.Gormigrate(sqlDB)
if err := m.MigrateTo("202010251337"); err != nil {
log.Fatalln(err)
}
// Migrate all files in filestorage
if err := sqlDB.Transaction(func(sqlTx *gorm.DB) error {
return boltDB.Bolt.View(func(boltTx *bolt.Tx) error {
// Migrate all the file uploads
allFUs, err := boltdb.AllFileUploads(boltTx)
if err != nil {
return err
}
var fusDeleted uint
var sqlFUs []db.FileUpload
for _, fu := range allFUs {
isDeleted := fu.State == boltdb.FileUploadStateDeleted
sqlFU := db.FileUpload{
State: db.FileUploadState(int(fu.State)),
PubID: fu.ID,
FileName: fu.FileName,
ContentType: fu.ContentType,
Checksum: fu.Checksum,
DeletedAt: deletedAt(isDeleted),
}
sqlFUs = append(sqlFUs, sqlFU)
if isDeleted {
fusDeleted++
}
}
log.Printf("migrating %v file uploads (of which %v deleted)", len(sqlFUs), fusDeleted)
sqlTx.Create(sqlFUs)
// Migrate all the pastes.
allPastes, err := boltdb.AllPastes(boltTx)
if err != nil {
return err
}
var pastesDeleted uint
var sqlPastes []db.Paste
for _, paste := range allPastes {
isDeleted := paste.State == boltdb.PasteStateDeleted
sqlPaste := db.Paste{
Type: db.PasteType(int(paste.Type)),
State: db.PasteState(int(paste.State)),
Content: paste.Content,
Key: paste.Key,
DeleteToken: paste.DeleteToken,
CreatedAt: paste.TimeCreated,
DeletedAt: deletedAt(isDeleted),
}
sqlPastes = append(sqlPastes, sqlPaste)
if isDeleted {
pastesDeleted++
}
}
log.Printf("migrating %v pastes (of which %v deleted)", len(sqlPastes), pastesDeleted)
sqlTx.Create(sqlPastes)
return nil
})
}); err != nil {
log.Fatalln(err)
}
log.Println("migration successful! :D")
}
func deletedAt(isDeleted bool) gorm.DeletedAt {
if isDeleted {
return gorm.DeletedAt{Time: time.Now(), Valid: true}
}
return gorm.DeletedAt{}
}

1
go.mod
View File

@ -10,7 +10,6 @@ require (
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.10.0 github.com/prometheus/client_golang v1.10.0
github.com/prometheus/common v0.23.0 // indirect github.com/prometheus/common v0.23.0 // indirect
go.etcd.io/bbolt v1.3.5
golang.org/x/sys v0.0.0-20210507161434-a76c4d0a0096 // indirect golang.org/x/sys v0.0.0-20210507161434-a76c4d0a0096 // indirect
gorm.io/driver/postgres v1.1.0 gorm.io/driver/postgres v1.1.0
gorm.io/driver/sqlite v1.1.4 gorm.io/driver/sqlite v1.1.4

3
go.sum
View File

@ -382,8 +382,6 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@ -471,7 +469,6 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

204
internal/boltdb/db.go
View File

@ -1,204 +0,0 @@
package boltdb
import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"os"
"time"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
)
// Database is the main rushlink database type.
//
// Open a database using DB.Open() and close it in the end using DB.Close().
// Only one instance of DB should exist in a program at any moment.
type Database struct {
Bolt *bolt.DB
}
// CurrentMigrateVersion holds the current "migrate version".
//
// If we alter the database format, we bump this number and write a new
// database migration in migrate().
const CurrentMigrateVersion = 3
// BucketConf holds the name for the "configuration" bucket.
//
// This bucket holds the database version, secret site-wide keys, etc.
const BucketConf = "conf"
// BucketPastes holds the name for the pastes bucket.
const BucketPastes = "pastes"
// BucketFileUpload holds the name for the file-upload bucket.
const BucketFileUpload = "fileUpload"
// KeyMigrateVersion stores the current migration version. If this value is less than
// CurrentMigrateVersion, the database has to be migrated.
const KeyMigrateVersion = "migrate_version"
// OpenDB opens a database file located at path.
func OpenDB(path string, fs *FileStore) (*Database, error) {
if path == "" {
return nil, errors.New("database not set")
}
db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return nil, errors.Wrapf(err, "failed to open database at '%v'", path)
}
if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil {
return nil, err
}
return &Database{db}, nil
}
// Close the bolt database
func (db *Database) Close() error {
if db == nil {
panic("no open database")
}
return db.Bolt.Close()
}
// Initialize and migrate the database to the current version
func migrate(tx *bolt.Tx, fs *FileStore) error {
// Guidelines for error handling:
// - Errors based on malformed *structure* should be fatal!
// - Errors based on malformed *data* should print a warning
// (and if possible try to fix the error).
dbVersion, err := dbVersion(tx)
if err != nil {
return err
}
// Migrate the database to version 1
if dbVersion < 1 {
log.Println("migrating database to version 1")
// Create conf bucket
_, err := tx.CreateBucket([]byte(BucketConf))
if err != nil {
return err
}
// Create paste bucket
_, err = tx.CreateBucket([]byte(BucketPastes))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 1); err != nil {
return err
}
}
if dbVersion < 2 {
log.Println("migrating database to version 2")
// Create fileUpload bucket
_, err := tx.CreateBucket([]byte(BucketFileUpload))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 2); err != nil {
return err
}
}
if dbVersion < 3 {
log.Println("migrating database to version 3")
// In this version, we changed te way how Content-Types are being
// stored. Previously, we allowed clients to provide their own
// Content-Types for files, using the Content-Disposition header in
// multipart forms. The new way detects these types using
// http.DetectContentType.
//
// Scan through all the FileUploads and update their ContentTypes.
bucket := tx.Bucket([]byte(BucketFileUpload))
cursor := bucket.Cursor()
var id, storedBytes []byte
id, storedBytes = cursor.First()
for id != nil {
fu, err := decodeFileUpload(storedBytes)
if err != nil {
log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id))
id, storedBytes = cursor.Next()
continue
}
if fu.State != FileUploadStatePresent {
id, storedBytes = cursor.Next()
continue
}
filePath := fu.Path(fs)
file, err := os.Open(fu.Path(fs))
if err != nil {
log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath))
id, storedBytes = cursor.Next()
continue
}
var buf bytes.Buffer
buf.Grow(512)
io.CopyN(&buf, file, 512)
contentType := http.DetectContentType(buf.Bytes())
if contentType != fu.ContentType {
fu.ContentType = contentType
fu.Save(tx)
cursor.Seek(id)
}
id, storedBytes = cursor.Next()
}
// Update the version number
if err := setDBVersion(tx, 3); err != nil {
return err
}
}
return nil
}
// Get the current migrate version from the database
func dbVersion(tx *bolt.Tx) (int, error) {
conf := tx.Bucket([]byte(BucketConf))
if conf == nil {
return 0, nil
}
dbVersionBytes := conf.Get([]byte(KeyMigrateVersion))
if dbVersionBytes == nil {
return 0, nil
}
// Version was already stored
var dbVersion int
if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil {
return 0, err
}
if dbVersion == 0 {
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
}
if dbVersion > CurrentMigrateVersion {
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion)
}
return dbVersion, nil
}
// Update the current migrate version in the database
func setDBVersion(tx *bolt.Tx, version int) error {
conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf))
if err != nil {
return err
}
versionBytes, err := gobmarsh.Marshal(version)
if err != nil {
return err
}
return conf.Put([]byte(KeyMigrateVersion), versionBytes)
}

256
internal/boltdb/fileupload.go
View File

@ -1,256 +0,0 @@
package boltdb
import (
"bytes"
"encoding/hex"
"hash/crc32"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
)
// Use the Castagnoli checksum because of the acceleration on Intel CPUs
var checksumTable = crc32.MakeTable(crc32.Castagnoli)
// FileStore holds the path to a file storage location.
type FileStore struct {
path string
}
// FileUploadState determines the current state of a FileUpload object.
type FileUploadState int
// FileUpload models an uploaded file.
type FileUpload struct {
// State of the FileUpload (present/deleted/etc).
State FileUploadState
// ID identifies this FileUpload.
ID uuid.UUID
// FileName contains the original filename of this FileUpload.
FileName string
// Content type as determined by http.DetectContentType.
ContentType string
// Checksum holds a crc32c checksum of the file.
//
// This checksum is only meant to allow for the detection of random
// database corruption.
Checksum uint32
}
const (
dirMode os.FileMode = 0750
fileMode os.FileMode = 0640
)
const (
// FileUploadStateUndef is an undefined FileUpload.
FileUploadStateUndef FileUploadState = 0
// FileUploadStatePresent denotes the normal (existing) state.
FileUploadStatePresent FileUploadState = 1
// FileUploadStateDeleted denotes a deleted state.
FileUploadStateDeleted FileUploadState = 2
)
func (t FileUploadState) String() string {
switch t {
case FileUploadStateUndef:
return "unknown"
case FileUploadStatePresent:
return "present"
case FileUploadStateDeleted:
return "deleted"
default:
return "invalid"
}
}
// OpenFileStore opens the file storage at path.
func OpenFileStore(path string) (*FileStore, error) {
if path == "" {
return nil, errors.New("file-store not set")
}
// Try to create the file store directory if it does not yet exist
if err := os.MkdirAll(path, dirMode); err != nil {
return nil, errors.Wrap(err, "creating file store directory")
}
return &FileStore{path[:]}, nil
}
// Path returns the path of the FileStore root.
func (fs *FileStore) Path() string {
return fs.path
}
// filePath resolves the path of a file in the FileStore given some id and filename.
func (fs *FileStore) filePath(id uuid.UUID, fileName string) string {
if fs.path == "" {
panic("fileStoreDir called while the file store path has not been set")
}
return path.Join(fs.path, hex.EncodeToString(id[:]), fileName)
}
// NewFileUpload creates a new FileUpload object.
//
// Internally, this function detects the type of the file stored in `r` using
// `http.DetectContentType`.
func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) {
// Generate a file ID
id, err := uuid.NewRandom()
if err != nil {
return nil, errors.Wrap(err, "generating UUID")
}
// Construct a checksum for this file
hash := crc32.New(checksumTable)
tee := io.TeeReader(r, hash)
// Detect the file type
var tmpBuf bytes.Buffer
tmpBuf.Grow(512)
io.CopyN(&tmpBuf, tee, 512)
contentType := http.DetectContentType(tmpBuf.Bytes())
// Open the file on disk for writing
baseName := filepath.Base(fileName)
filePath := fs.filePath(id, baseName)
if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil {
return nil, errors.Wrap(err, "creating file dir")
}
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode)
if err != nil {
return nil, errors.Wrap(err, "opening file")
}
defer file.Close()
// Write the file to disk
_, err = io.Copy(file, &tmpBuf)
if err != nil {
return nil, errors.Wrap(err, "writing to file")
}
_, err = io.Copy(file, tee)
if err != nil {
return nil, errors.Wrap(err, "writing to file")
}
fu := &FileUpload{
State: FileUploadStatePresent,
ID: id,
FileName: baseName,
ContentType: contentType,
Checksum: hash.Sum32(),
}
return fu, nil
}
// GetFileUpload tries to retrieve a FileUpload object from the bolt database.
func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
storedBytes := bucket.Get(id[:])
if storedBytes == nil {
return nil, nil
}
return decodeFileUpload(storedBytes)
}
// AllFileUploads tries to retrieve all FileUpload objects from the bolt database.
func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
var fus []FileUpload
err := bucket.ForEach(func(_, storedBytes []byte) error {
fu, err := decodeFileUpload(storedBytes)
if err != nil {
return err
}
fus = append(fus, *fu)
return nil
})
if err != nil {
return nil, err
}
return fus, nil
}
func decodeFileUpload(storedBytes []byte) (*FileUpload, error) {
fu := &FileUpload{}
err := gobmarsh.Unmarshal(storedBytes, fu)
return fu, err
}
// Save saves a FileUpload in the database.
func (fu *FileUpload) Save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
buf, err := gobmarsh.Marshal(fu)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put(fu.ID[:], buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
}
// Delete deletes a FileUpload from the database.
func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
// Remove the file in the backend
filePath := fu.Path(fs)
if err := os.Remove(filePath); err != nil {
return err
}
// Update the file in the server
if err := (&FileUpload{
ID: fu.ID,
State: FileUploadStateDeleted,
}).Save(tx); err != nil {
return err
}
// Cleanup the parent directory
wrap := "deletion succeeded, but removing the file directory has failed"
return errors.Wrap(os.Remove(path.Dir(filePath)), wrap)
}
// Path returns the path to this FileUpload in the FileStore provided in fs.
func (fu *FileUpload) Path(fs *FileStore) string {
return fs.filePath(fu.ID, fu.FileName)
}
// URL returns the URL for the FileUpload.
func (fu *FileUpload) URL() *url.URL {
rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName
urlParse, err := url.Parse(rawurl)
if err != nil {
panic("could not construct /uploads/ url")
}
return urlParse
}
// Ext returns the extension of the file attached to this FileUpload.
func (fu *FileUpload) Ext() string {
return filepath.Ext(fu.FileName)
}

345
internal/boltdb/paste.go
View File

@ -1,345 +0,0 @@
package boltdb
import (
"crypto/rand"
"encoding/base64"
"encoding/hex"
"net/http"
"net/url"
"strings"
"time"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
// PasteType describes the type of Paste (i.e. file, redirect, [...]).
type PasteType int
// PasteState describes the state of a Paste (i.e. present, deleted, [...]).
type PasteState int
// Paste describes the main Paste model in the database.
type Paste struct {
Type PasteType
State PasteState
Content []byte
Key string
DeleteToken string
TimeCreated time.Time
}
// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen
var ReservedPasteKeys = []string{"xd42", "example"}
// Note: we use iota here. That means removals of PasteType* are not allowed,
// because this changes the value of the constant. Please add the comment
// "// deprecated" if you want to remove the constant. Additions are only
// allowed at the bottom of this block, for the same reason.
const (
PasteTypeUndef PasteType = iota
// PasteTypePaste is as of yet unused. It is still unclear if this type
// will ever get a proper meaning.
PasteTypePaste
PasteTypeRedirect
PasteTypeFileUpload
)
// Note: we use iota here. See the comment above PasteType*
const (
PasteStateUndef PasteState = iota
PasteStatePresent
PasteStateDeleted
)
// minKeyLen specifies the mimimum length of a paste key.
const minKeyLen = 4
var (
// ErrKeyInvalidChar occurs when a key contains an invalid character.
ErrKeyInvalidChar = errors.New("invalid character in key")
// ErrKeyInvalidLength occurs when a key embeds a length that is incorrect.
ErrKeyInvalidLength = errors.New("key length encoding is incorrect")
// ErrPasteDoesNotExist occurs when a key does not exist in the database.
ErrPasteDoesNotExist = errors.New("url key not found in the database")
)
// ErrHTTPStatusCode returns the HTTP status code that should correspond to
// the provided error.
// server error, or false if it is not.
func ErrHTTPStatusCode(err error) int {
switch err {
case nil:
return 0
case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist:
return http.StatusNotFound
}
return http.StatusInternalServerError
}
// Base64 encoding and decoding
var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding)
func (t PasteType) String() string {
switch t {
case PasteTypeUndef:
return "unknown"
case PasteTypePaste:
return "paste"
case PasteTypeRedirect:
return "redirect"
case PasteTypeFileUpload:
return "file"
default:
return "invalid"
}
}
func (t PasteState) String() string {
switch t {
case PasteStateUndef:
return "unknown"
case PasteStatePresent:
return "present"
case PasteStateDeleted:
return "deleted"
default:
return "invalid"
}
}
// GetPaste retrieves a paste from the database.
func GetPaste(tx *bolt.Tx, key string) (*Paste, error) {
if err := ValidatePasteKey(key); err != nil {
return nil, err
}
return GetPasteNoValidate(tx, key)
}
// ValidatePasteKey validates the format of the key that has
func ValidatePasteKey(key string) error {
internalLen := minKeyLen
countingOnes := true
for _, ch := range key {
limb := strings.IndexRune(base64Alphabet, ch)
if limb == -1 {
return ErrKeyInvalidChar
}
for i := 5; i >= 0 && countingOnes; i-- {
if (limb>>uint(i))&0x1 == 0 {
countingOnes = false
break
}
internalLen++
}
}
if internalLen != len(key) {
return ErrKeyInvalidLength
}
return nil
}
// GetPasteNoValidate retrieves a paste from the database without validating
// the key format first.
func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) {
pastesBucket := tx.Bucket([]byte(BucketPastes))
if pastesBucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
}
storedBytes := pastesBucket.Get([]byte(key))
if storedBytes == nil {
return nil, ErrPasteDoesNotExist
}
return decodePaste(storedBytes)
}
func decodePaste(storedBytes []byte) (*Paste, error) {
p := &Paste{}
err := gobmarsh.Unmarshal(storedBytes, p)
return p, err
}
// Save saves this Paste to the database.
func (p *Paste) Save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketPastes))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BucketPastes)
}
buf, err := gobmarsh.Marshal(p)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put([]byte(p.Key), buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
}
// Delete deletes this Paste from the database.
func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error {
// Remove the (maybe) attached file
if p.Type == PasteTypeFileUpload {
fuID, err := uuid.FromBytes(p.Content)
if err != nil {
return errors.Wrap(err, "failed to parse uuid")
}
fu, err := GetFileUpload(tx, fuID)
if err != nil {
return errors.Wrap(err, "failed to find file in database")
}
if err := fu.Delete(tx, fs); err != nil {
return errors.Wrap(err, "failed to remove file")
}
}
// Replace the old paste with a new empty paste
p.Type = PasteTypeUndef
p.State = PasteStateDeleted
p.Content = []byte{}
if err := p.Save(tx); err != nil {
return errors.Wrap(err, "failed to delete paste in database")
}
return nil
}
// RedirectURL returns the URL from this paste.
//
// This function assumes that the paste is valid. If the paste struct is
// corrupted in some way, this function will panic.
func (p *Paste) RedirectURL() *url.URL {
if p.Type != PasteTypeRedirect {
panic("expected p.Type to be PasteTypeRedirect")
}
rawurl := string(p.Content)
urlParse, err := url.Parse(rawurl)
if err != nil {
panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key))
}
return urlParse
}
// GeneratePasteKey generates a new paste key. It will ensure that the newly
// generated paste key does not already exist in the database.
// The running time of this function is in O(log N), where N is the amount of
// keys stored in the url-shorten database.
// In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum
// guessing entropy of the generated key.
func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) {
pastesBucket := tx.Bucket([]byte(BucketPastes))
if pastesBucket == nil {
return "", errors.Errorf("bucket %v does not exist", BucketPastes)
}
epoch := 0
var key string
for {
var err error
key, err = generatePasteKeyInner(epoch, minimumEntropy)
if err != nil {
return "", errors.Wrap(err, "url-key generation failed")
}
found := pastesBucket.Get([]byte(key))
if found == nil {
break
}
isReserved := false
for _, reservedKey := range ReservedPasteKeys {
if strings.HasPrefix(key, reservedKey) {
isReserved = true
break
}
}
if !isReserved {
break
}
epoch++
}
return key, nil
}
// generatePasteKeyInner generates a new paste key, but leaves the
// uniqueness and is-reserved checks to the caller. That is, it only
// generates a random key in the correct (syntactical) format.
// Both epoch and entropy can be used to set the key length. Epoch is used
// to prevent collisions in retrying to generate new keys. Entropy (in bits)
// is used to ensure that a new key has at least some amount of guessing
// entropy.
func generatePasteKeyInner(epoch, entropy int) (string, error) {
entropyEpoch := entropy
entropyEpoch -= minKeyLen * 6 // First 4 characters provide 24 bits.
entropyEpoch++ // One bit less because of '0' bit.
entropyEpoch = (entropyEpoch-1)/5 + 1 // 5 bits for every added epoch.
if epoch < entropyEpoch {
epoch = entropyEpoch
}
urlKey := make([]byte, minKeyLen+epoch)
_, err := rand.Read(urlKey)
if err != nil {
return "", err
}
// Put all the values in the range 0..64 for easier base64-encoding
for i := 0; i < len(urlKey); i++ {
urlKey[i] &= 0x3F
}
// Implement truncate-resistance by forcing the prefix to
// 0b111110xxxxxxxxxx
// ^----- {epoch} ones followed by a single 0
//
// Example when epoch is 1: prefix is 0b10.
i := 0
for i < epoch {
// Set this bit to 1
limb := i / 6
bit := i % 6
urlKey[limb] |= 1 << uint(5-bit)
i++
}
// Finally set the next bit to 0
limb := i / 6
bit := i % 6
urlKey[limb] &= ^(1 << uint(5-bit))
// Convert this ID to a canonical base64 notation
for i := range urlKey {
urlKey[i] = base64Alphabet[urlKey[i]]
}
return string(urlKey), nil
}
// GenerateDeleteToken generates a new (random) delete token.
func GenerateDeleteToken() (string, error) {
var deleteToken [16]byte
_, err := rand.Read(deleteToken[:])
if err != nil {
return "", err
}
return hex.EncodeToString(deleteToken[:]), nil
}
// AllPastes tries to retrieve all the Paste objects from the database.
func AllPastes(tx *bolt.Tx) ([]Paste, error) {
bucket := tx.Bucket([]byte(BucketPastes))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
}
var ps []Paste
err := bucket.ForEach(func(_, storedBytes []byte) error {
p, err := decodePaste(storedBytes)
if err != nil {
return err
}
ps = append(ps, *p)
return nil
})
if err != nil {
return nil, err
}
return ps, nil
}

38
internal/boltdb/paste_test.go
View File

@ -1,38 +0,0 @@
package boltdb
import "testing"
func TestValidatePasteKey(t *testing.T) {
tests := []struct {
key string
wantErr bool
errKind error
}{
{"xd42__", false, nil},
{"xd42_*", true, ErrKeyInvalidChar},
{"xd42_/", true, ErrKeyInvalidChar},
{"xd42_=", true, ErrKeyInvalidChar},
{"xd42_", true, ErrKeyInvalidLength},
{"xd42", true, ErrKeyInvalidLength},
{"xd4", true, ErrKeyInvalidLength},
{"xd", true, ErrKeyInvalidLength},
{"x", true, ErrKeyInvalidLength},
{"", true, ErrKeyInvalidLength},
{"KoJ5", false, nil},
{"__dGSJIIbBpr-SD0", false, nil},
{"__dGSJIIbBpr-SD", true, ErrKeyInvalidLength},
}
for _, tt := range tests {
t.Run(tt.key, func(t *testing.T) {
err := ValidatePasteKey(tt.key)
if (err != nil) != tt.wantErr {
t.Errorf("ValidatePasteKey() got error = %v, want error %v", err != nil, tt.wantErr)
}
if (err != nil) && err != tt.errKind {
t.Errorf("ValidatePasteKey() error = %v, want errKind %v", err, tt.errKind)
}
})
}
}