electricdusk/rushlink
3
3
Fork 3
Code Issues 21 Pull Requests 2 Releases Activity

Use SQL database instead of bolt #71

Merged
mrngm merged 6 commits from sql into master 2021-05-05 21:34:29 +02:00
Conversation 0 Commits 6 Files Changed 19 +1414 -353
19 changed files with 1414 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
assets/templates/html/pasteMeta.html.tmpl
View File

@ -20,10 +20,18 @@ with delete token: <unknown>
{{end -}}
type: {{.Paste.Type}}
state: {{.Paste.State}}
{{if .Paste.TimeCreated.IsZero -}}
{{if .Paste.CreatedAt.IsZero -}}
created: unknown
{{else -}}
created: {{.Paste.TimeCreated}}
created: {{.Paste.CreatedAt}}
{{end -}}
{{if .Paste.UpdatedAt.IsZero -}}
last updated: unknown
{{else -}}
last updated: {{.Paste.UpdatedAt}}
{{end -}}
{{if .Paste.DeletedAt.Valid -}}
deleted at: {{.Paste.DeletedAt.Time}}
{{end -}}
delete token: {{.CanDeleteString}}
<form action="delete?deleteToken={{.Paste.DeleteToken}}" method="post">

4
assets/templates/txt/pasteMeta.txt.tmpl
View File

@ -6,10 +6,10 @@ with delete token: {{.RootURL}}/{{.Paste.Key}}{{.FileExt}}?deleteToken={{.Paste.
with delete token: <unknown>
{{end -}}type: {{.Paste.Type}}
state: {{.Paste.State}}
{{if .Paste.TimeCreated.IsZero -}}
{{if .Paste.CreatedAt.IsZero -}}
created: unknown
{{else -}}
created: {{.Paste.TimeCreated}}
created: {{.Paste.CreatedAt}}
{{end -}}
delete token: {{.CanDeleteString}}

20
bindata.go
View File

@ -102,7 +102,7 @@ func cssMainCss() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "css/main.css", size: 512, mode: os.FileMode(420), modTime: time.Unix(1587564030, 0)}
info := bindataFileInfo{name: "css/main.css", size: 512, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
@ -122,7 +122,7 @@ func jsCopyclipboardJs() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "js/copyclipboard.js", size: 1593, mode: os.FileMode(420), modTime: time.Unix(1587564030, 0)}
info := bindataFileInfo{name: "js/copyclipboard.js", size: 1593, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
@ -142,7 +142,7 @@ func jsDragdropJs() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "js/dragdrop.js", size: 2066, mode: os.FileMode(420), modTime: time.Unix(1576493365, 0)}
info := bindataFileInfo{name: "js/dragdrop.js", size: 2066, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
@ -162,7 +162,7 @@ func templatesHtmlBaseHtmlTmpl() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "templates/html/base.html.tmpl", size: 365, mode: os.FileMode(420), modTime: time.Unix(1576604408, 0)}
info := bindataFileInfo{name: "templates/html/base.html.tmpl", size: 365, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
@ -202,12 +202,12 @@ func templatesHtmlIndexHtmlTmpl() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "templates/html/index.html.tmpl", size: 1786, mode: os.FileMode(420), modTime: time.Unix(1576794526, 0)}
info := bindataFileInfo{name: "templates/html/index.html.tmpl", size: 1786, mode: os.FileMode(420), modTime: time.Unix(1595849380, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var _templatesHtmlPastemetaHtmlTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x94\xc1\x6e\xdb\x30\x0c\x86\xef\x7a\x0a\x42\x87\x36\x3d\x38\xbe\xb7\xb6\x07\x2c\xdd\x80\x61\x3d\x0c\x5d\x7b\xd9\x4d\xb1\x98\x5a\x8d\x22\x69\x12\xdd\xd6\x10\xf4\xee\x83\xec\x3a\xe9\xb6\x60\x4b\x2f\x01\x42\x51\x1f\xff\x9f\x22\x1d\xa3\xc4\x8d\x32\x08\x9c\x14\x69\xe4\x29\xb1\xf3\x18\x97\xdf\x44\x20\x5c\x7e\xc5\x21\xa5\x18\x97\x9f\x95\xc6\x4f\x2f\x94\xd2\x39\xec\x90\x84\x14\x24\xa0\x00\xdf\x87\x4e\x2b\xb3\x65\x31\xa2\x91\x29\x31\x76\x80\x75\x28\x64\x21\x9c\x43\x23\x33\xb2\x0a\xad\x57\x8e\x80\x06\x87\x35\x27\x7c\xa1\xf2\x51\x3c\x89\x29\xca\x21\xf8\xb6\xe6\xe5\x63\x28\x5b\xeb\x86\x56\x2b\xb7\xb6\xc2\xcb\xe5\x63\xe0\x20\x71\x83\xbe\xa9\xca\x29\xb5\x39\x56\x6b\x6d\xe5\x30\x16\x71\x1e\x41\xc9\x9a\x67\xca\x9d\x5d\xcd\x9c\x95\x35\x24\x94\x41\xcf\xa1\xd5\x22\x84\x9a\x77\x4a\x4a\x34\xbc\x61\xd5\xba\x27\xb2\x06\xac\x69\xb5\x6a\xb7\x7f\x5d\x5d\xe4\x5e\xdc\x5a\x4b\xf7\xb7\x37\x29\x95\xff\x68\xcc\x05\x6f\x56\xd6\x0d\x70\x7f\x7b\x03\x64\x61\x6f\xa2\x2a\xa7\x12\x0d\x54\xc1\x09\x73\x4c\xdf\x77\x12\xd4\x07\x9e\x4d\x3a\x61\x1a\x56\x95\xce\x63\x33\xda\x69\x58\x25\xa0\xf3\xb8\xa9\xf9\x89\x42\x78\x73\x62\x62\x55\x8a\x86\x15\x45\xc1\x62\x54\x1b\x10\x46\xc2\xc2\x20\xbc\x66\x67\x49\xf9\xd7\x2b\xf3\x00\x5c\xa2\x46\x42\xc9\x2f\x60\xb9\x12\xe6\x7a\xfc\xf7\xd1\x5a\x9d\x12\x7b\x56\xd4\xc1\x74\x0e\x64\xb7\x68\x2e\xe1\xbd\x8a\x3f\x4c\xd7\xef\xf2\xed\x7a\x9f\x76\x7d\x08\xbe\xc3\xd4\x09\xac\xd1\x77\x8c\xa8\x03\x42\x71\xdc\xc1\x99\xa6\xab\xde\x6c\x8d\x7d\x36\x67\x0f\x74\x35\xcd\xdc\x98\x9c\xe7\xf7\x12\xf6\xe0\xbb\xc1\x61\x4a\x2c\xe4\x6e\xbd\x09\x8f\xdd\x4b\x69\xea\xec\x9c\xaa\x76\xb8\xf2\x28\x08\xe5\xf2\x4b\xf8\x81\xde\x8e\xc0\x76\x0a\x5d\xc2\x6b\xbd\xb7\xca\xf6\x67\x87\x7a\x07\xc8\x88\x9f\x65\xfd\x2e\x3f\xc6\xc3\x33\x4d\x4f\x98\x97\x63\x63\xfd\x0e\x44\x4b\xca\x9a\xfa\xf5\x45\x4f\xe9\x7c\x5e\xf8\xce\xca\x9a\x3b\x1b\x28\x6f\x8c\x32\xae\x9f\x17\x39\xf4\xeb\x9d\x22\x0e\x46\xec\x70\x86\x72\x78\x12\xba\xc7\x9a\x4f\x1c\x70\x19\xca\x59\x8c\x05\xa8\x0d\x58\x0f\x0b\xfc\xf9\xbf\x31\x5b\x18\x4b\x7f\xcc\xda\xc5\x64\x54\x05\xb1\xd6\x28\x47\xdc\xec\x3e\x6f\x4c\x76\xb7\xdf\x9c\xf9\x1b\xf1\x2b\x00\x00\xff\xff\xf7\x78\x59\x94\xdc\x04\x00\x00")
var _templatesHtmlPastemetaHtmlTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x94\xc1\x6e\xdb\x38\x10\x86\xef\x7a\x8a\x01\x0f\x89\x73\x90\x75\x4f\x24\x2d\x76\x9d\x5d\x60\xd1\x1c\x8a\xd4\xe9\xa1\x37\x5a\x1c\xc7\x8c\x69\x92\x25\x47\x49\x0c\x82\xef\x5e\x50\xb4\x65\xa7\x71\x5b\xe7\x22\x40\xc3\xe1\xf7\xcf\x3f\x98\x61\x08\x02\x97\x52\x23\x30\x92\xa4\x90\xc5\x58\x5c\x86\x30\xfd\xcc\x3d\xe1\xf4\x13\x6e\x63\x0c\x61\xfa\x9f\x54\xf8\xef\x2b\xc5\x78\x09\x1b\x24\x2e\x38\x71\x28\xc1\xf5\x7e\xa5\xa4\x5e\x17\x21\xa0\x16\x31\x16\xc5\x01\xb6\x42\x2e\x4a\x6e\x2d\x6a\x91\x90\xb5\xef\x9c\xb4\x04\xb4\xb5\xd8\x30\xc2\x57\xaa\x9e\xf8\x33\xcf\x51\x06\xde\x75\x0d\xab\x9e\x7c\xd5\x19\xbb\xed\x94\xb4\x0b\xc3\x9d\x98\x3e\x79\x06\x02\x97\xe8\xda\xba\xca\xa9\xed\x29\xad\x85\x11\xdb\x41\xc4\x3a\x04\x29\x1a\x96\x28\x73\x33\xdb\x73\x66\x46\x13\x97\x1a\x1d\x83\x4e\x71\xef\x1b\xb6\x92\x42\xa0\x66\x6d\x51\x2f\x7a\x22\xa3\xc1\xe8\x4e\xc9\x6e\xfd\xee\xea\x24\xf5\xe2\xde\x18\x7a\xb8\xbf\x8b\xb1\xfa\x4d\x63\xae\x58\x3b\x33\x76\x0b\x0f\xf7\x77\x40\x06\x46\x13\x75\x95\x25\x5a\xa8\xbd\xe5\xfa\x54\x7d\x5f\x88\x53\xef\x59\x32\x69\xb9\x6e\x8b\xba\xb2\x0e\xdb\xc1\x4e\x5b\xd4\x1c\x56\x0e\x97\x0d\x3b\xb3\x10\xd6\x9e\x99\x58\x57\xbc\x2d\xca\xb2\x2c\x42\x90\x4b\xe0\x5a\xc0\x44\x23\xec\xb2\x53\x49\xe9\xeb\xa4\x7e\x04\x26\x50\x21\xa1\x60\x57\x30\x9d\x71\x7d\x3b\xfc\xfd\x63\x8c\x8a\xb1\x78\x91\xb4\x82\x7c\x0e\x64\xd6\xa8\xaf\xe1\xa3\x15\xff\x95\xaf\xcf\xd3\xed\x66\x4c\xbb\x3d\x04\x3f\x60\xea\x0c\xd6\xe0\x3b\x04\x54\x1e\xa1\x3c\xed\xe0\x42\xd1\x4d\xaf\xd7\xda\xbc\xe8\x8b\x47\xba\xc9\x33\x37\x24\xa7\xf9\xbd\x86\x11\x3c\xdf\x5a\x8c\xb1\xf0\xa9\x5b\x47\xe1\xa1\x7b\x31\xe6\xce\xee\x62\x33\x87\x9c\x50\xfc\x4d\xd3\xff\xfd\x37\x74\x66\xc0\x75\x39\x78\x0d\x3b\xb5\xe3\xba\xc6\xb3\x11\x3b\x22\x06\xf4\xbe\xa4\x63\x91\x07\x2b\xde\x8b\x28\xee\x09\xfa\x7c\x72\x52\xe9\x6d\xc2\x28\x37\xc2\x7e\x29\x97\xfb\x9a\xe4\xbe\x72\x25\xf3\xf9\x6e\x56\x80\xd3\x11\xea\x90\x38\x97\x1b\x7c\xc3\x7b\xdb\xf9\x10\x0e\x13\x96\xa7\x2f\xed\xf5\xd2\xb8\x0d\xf0\x8e\xa4\xd1\xcd\x6e\x18\xcf\x19\x9a\xf4\x56\xad\x8c\x68\x98\x35\x9e\xd2\xb2\x4b\x6d\xfb\xfd\x1b\xe4\xfb\xc5\x46\x12\x03\xcd\x37\xb8\x87\x32\x78\xe6\xaa\xc7\x86\x65\x0e\xd8\x04\x65\x45\x08\x25\xc8\x25\x18\x07\x13\xfc\xfe\xa7\x0d\x99\x68\x43\x3f\xad\xc9\x55\x36\x2a\x3d\x5f\x28\x14\x03\x6e\xef\x3e\x2d\x7b\x72\x37\x2e\xfd\xfe\x79\xfb\x11\x00\x00\xff\xff\x3a\xc6\xf5\xbb\x97\x05\x00\x00")
func templatesHtmlPastemetaHtmlTmplBytes() ([]byte, error) {
return bindataRead(
@ -222,7 +222,7 @@ func templatesHtmlPastemetaHtmlTmpl() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "templates/html/pasteMeta.html.tmpl", size: 1244, mode: os.FileMode(420), modTime: time.Unix(1589228935, 0)}
info := bindataFileInfo{name: "templates/html/pasteMeta.html.tmpl", size: 1431, mode: os.FileMode(420), modTime: time.Unix(1603641311, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
@ -282,12 +282,12 @@ func templatesTxtIndexTxtTmpl() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "templates/txt/index.txt.tmpl", size: 684, mode: os.FileMode(420), modTime: time.Unix(1576794536, 0)}
info := bindataFileInfo{name: "templates/txt/index.txt.tmpl", size: 684, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
var _templatesTxtPastemetaTxtTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x51\xcd\x4a\x03\x31\x10\xbe\xe7\x29\x86\x78\x51\x30\xf1\x5e\xfc\x01\xdb\x15\xc4\x1e\xa4\xae\x17\x4f\x5d\x76\xa7\x36\x74\x4d\xea\x66\x96\x76\x09\xf3\xee\xb2\x49\xdd\x2d\xea\x41\xd0\x4b\x20\x33\xdf\xcc\xf7\x33\x21\xe8\x85\x73\xf4\xbc\x98\x33\x5f\x84\xa0\x1f\x0b\x4f\xa8\x1f\xb0\x63\x0e\x41\xdf\x99\x1a\xb3\x3d\x31\x0b\xa5\x94\x08\xc1\xac\xa0\xb0\x15\x9c\x5a\x84\x03\xf2\x89\x8a\xf8\x36\xc6\xbe\x82\xac\xb0\x46\xc2\x4a\x9e\x81\x9e\x16\x76\x16\x7f\xb7\xce\xd5\xcc\x62\x67\x68\x0d\xa9\x0f\xe4\x36\x68\x27\xf0\x4b\xee\x9b\x34\x95\xf7\x43\x57\x03\x6c\x36\x16\x99\x45\x08\x58\x7b\x04\xf5\x33\xd1\x65\x6b\x37\xd6\xed\xec\x75\x8f\xb3\x55\x0f\xa3\x6e\x8b\x51\x41\xda\x96\x77\x5b\x64\x16\xbe\x77\x73\x54\x8e\xee\xe2\x7a\xb3\xfa\x74\x9c\x9b\x37\x9c\x36\x58\x10\x56\xfa\xde\xbf\x60\xe3\x22\x6d\x99\x4a\x13\x38\x70\x1d\x4b\x1a\x7a\x23\xdf\xb8\x24\xa9\x4f\xaa\xc4\xb7\x80\x86\x18\x53\xc4\xcc\xe2\x6f\x67\x58\x2e\x97\xe2\x04\x72\x37\x24\xb4\x36\xfe\x6b\x0c\xe7\x80\x7b\x2c\x5b\xc2\x89\x28\xdb\xa6\x06\xa5\x1a\x7c\x6f\xd1\x13\xc8\x59\x36\xcf\xf2\x4c\x82\xfc\xbf\xdb\xc9\xa8\x29\x46\xc0\x2c\x3e\x02\x00\x00\xff\xff\x34\xf2\xf4\x10\x90\x02\x00\x00")
var _templatesTxtPastemetaTxtTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x51\x4f\x4b\x3b\x31\x10\xbd\xe7\x53\x0c\xf9\x5d\x7e\x82\x89\xf7\xe2\x1f\xb4\x5d\x41\xec\x41\xea\x7a\xf1\xd4\x65\x77\x6a\x43\x97\xa4\x6e\x66\x69\x97\x30\xdf\x5d\x36\xa9\xb1\xa8\x07\x41\x2f\x81\xcc\xbc\x79\xef\xcd\x9b\x10\xf4\xc2\x39\x7a\x5a\xcc\x99\xcf\x42\xd0\x0f\x95\x27\xd4\xf7\x38\x30\x87\xa0\x6f\x4d\x8b\xc5\x9e\x98\x85\x52\x4a\x84\x60\x56\x50\xd9\x06\xfe\x5b\x84\x03\xf2\x91\xaa\xf8\x76\xc6\xbe\x80\x6c\xb0\x45\xc2\x46\x9e\x80\x9e\x56\x76\x16\x7f\x37\xce\xb5\xcc\x62\x67\x68\x0d\xa9\x0f\xe4\x36\x68\x27\xf0\x43\xed\xab\x34\x55\x8e\x43\x17\x19\x36\xfb\x28\x32\x8b\x10\xb0\xf5\x08\xea\x7b\xa1\xf3\xde\x6e\xac\xdb\xd9\xcb\x11\x67\x9b\x11\x46\xc3\x16\xa3\x83\xc4\x56\x0e\x5b\x64\x16\x7e\xdc\xe6\xa8\x1c\xb7\x8b\xf4\x66\xf5\xbe\xf1\xb4\xc3\x8a\xb0\xb9\x26\x7d\xe7\x9f\xb1\x73\x51\xb4\x4e\xc5\x09\x1c\x94\x8e\x0d\xe5\x5e\xa6\xcd\x14\xc9\x79\x72\x24\xbe\x84\x93\x23\x4c\xf1\x32\x8b\xdf\x9d\x60\xb9\x5c\x8a\x7f\x50\xba\x9c\xce\xda\xf8\xcf\x11\x9c\x02\xee\xb1\xee\x09\x27\xa2\xee\xbb\x16\x94\xea\xf0\xb5\x47\x4f\x20\x67\xc5\xbc\x28\x0b\x09\xf2\xef\xee\x26\xa3\xa7\x18\x01\xb3\x78\x0b\x00\x00\xff\xff\x81\xcd\x43\x9e\x8c\x02\x00\x00")
func templatesTxtPastemetaTxtTmplBytes() ([]byte, error) {
return bindataRead(
@ -302,7 +302,7 @@ func templatesTxtPastemetaTxtTmpl() (*asset, error) {
return nil, err
}
info := bindataFileInfo{name: "templates/txt/pasteMeta.txt.tmpl", size: 656, mode: os.FileMode(420), modTime: time.Unix(1589229127, 0)}
info := bindataFileInfo{name: "templates/txt/pasteMeta.txt.tmpl", size: 652, mode: os.FileMode(420), modTime: time.Unix(1603638601, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}

110
cmd/rushlink-migrate-db/main.go Normal file
View File

@ -0,0 +1,110 @@
package main
import (
"flag"
"log"
"time"
"gitea.hashru.nl/dsprenkels/rushlink/internal/boltdb"
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
bolt "go.etcd.io/bbolt"
"gorm.io/gorm"
)
var (
boltDBPath = flag.String("boltdb", "", "location of the bolt database file")
fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored")
)
func main() {
flag.Parse()
boltFileStore, err := boltdb.OpenFileStore(*fileStorePath)
if err != nil {
log.Fatalln(err)
}
boltDB, err := boltdb.OpenDB(*boltDBPath, boltFileStore)
if err != nil {
log.Fatalln(err)
}
defer boltDB.Close()
sqlDB, err := db.OpenDBFromEnvironment()
if err != nil {
log.Fatalln(err)
}
// Migrate database schema
m := db.Gormigrate(sqlDB)
if err := m.MigrateTo("202010251337"); err != nil {
log.Fatalln(err)
}
// Migrate all files in filestorage
if err := sqlDB.Transaction(func(sqlTx *gorm.DB) error {
return boltDB.Bolt.View(func(boltTx *bolt.Tx) error {
// Migrate all the file uploads
allFUs, err := boltdb.AllFileUploads(boltTx)
if err != nil {
return err
}
var fusDeleted uint
var sqlFUs []db.FileUpload
for _, fu := range allFUs {
isDeleted := fu.State == boltdb.FileUploadStateDeleted
sqlFU := db.FileUpload{
State: db.FileUploadState(int(fu.State)),
PubID: fu.ID,
FileName: fu.FileName,
ContentType: fu.ContentType,
Checksum: fu.Checksum,
DeletedAt: deletedAt(isDeleted),
}
sqlFUs = append(sqlFUs, sqlFU)
if isDeleted {
fusDeleted++
}
}
log.Printf("migrating %v file uploads (of which %v deleted)", len(sqlFUs), fusDeleted)
sqlTx.Create(sqlFUs)
// Migrate all the pastes.
allPastes, err := boltdb.AllPastes(boltTx)
if err != nil {
return err
}
var pastesDeleted uint
var sqlPastes []db.Paste
for _, paste := range allPastes {
isDeleted := paste.State == boltdb.PasteStateDeleted
sqlPaste := db.Paste{
Type: db.PasteType(int(paste.Type)),
State: db.PasteState(int(paste.State)),
Content: paste.Content,
Key: paste.Key,
DeleteToken: paste.DeleteToken,
CreatedAt: paste.TimeCreated,
DeletedAt: deletedAt(isDeleted),
}
sqlPastes = append(sqlPastes, sqlPaste)
if isDeleted {
pastesDeleted++
}
}
log.Printf("migrating %v pastes (of which %v deleted)", len(sqlPastes), pastesDeleted)
sqlTx.Create(sqlPastes)
return nil
})
}); err != nil {
log.Fatalln(err)
}
log.Println("migration successful! :D")
}
func deletedAt(isDeleted bool) gorm.DeletedAt {
if isDeleted {
return gorm.DeletedAt{Time: time.Now(), Valid: true}
}
return gorm.DeletedAt{}
}

13
cmd/rushlink/main.go
View File

@ -6,11 +6,10 @@ import (
"gitea.hashru.nl/dsprenkels/rushlink"
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
"github.com/pkg/errors"
)
var (
databasePath = flag.String("database", "", "location of the database file")
fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored")
httpListen = flag.String("listen", "127.0.0.1:8000", "listen address (host:port)")
metricsListen = flag.String("metrics_listen", "127.0.0.1:58614", "listen address for metrics (host:port)")
rootURL = flag.String("root_url", "", "host root (example: 'https://example.com', uses an educated guess if omitted)")
@ -19,15 +18,19 @@ var (
func main() {
flag.Parse()
filestore, err := db.OpenFileStore(*fileStorePath)
filestore, err := db.OpenFileStoreFromEnvironment()
if err != nil {
log.Fatalln(err)
}
database, err := db.OpenDB(*databasePath, filestore)
database, err := db.OpenDBFromEnvironment()
if err != nil {
log.Fatalln(err)
}
defer database.Close()
migrate := db.Gormigrate(database)
if err := migrate.Migrate(); err != nil {
log.Fatal(errors.Wrap(err, "migrating database"))
}
go rushlink.StartMetricsServer(*metricsListen, database, filestore)
rushlink.StartMainServer(*httpListen, database, filestore, *rootURL)

4
go.mod
View File

@ -4,6 +4,7 @@ go 1.12
require (
github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect
github.com/go-gormigrate/gormigrate/v2 v2.0.0
github.com/golang/protobuf v1.4.2 // indirect
github.com/google/go-cmp v0.4.1 // indirect
github.com/google/uuid v1.1.1
@ -14,4 +15,7 @@ require (
go.etcd.io/bbolt v1.3.4
golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect
google.golang.org/protobuf v1.24.0 // indirect
gorm.io/driver/postgres v1.0.0
gorm.io/driver/sqlite v1.1.3
gorm.io/gorm v1.20.5
)

177
go.sum
View File

@ -1,9 +1,11 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@ -12,18 +14,30 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsdJRF26M=
github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE=
github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
github.com/go-gormigrate/gormigrate v1.6.0 h1:JvKpFsbVPLfIPPkIrZq1zq19PLFT4DVSnL5BiyE1P0E=
github.com/go-gormigrate/gormigrate/v2 v2.0.0 h1:e2A3Uznk4viUC4UuemuVgsNnvYZyOA8B3awlYk3UioU=
github.com/go-gormigrate/gormigrate/v2 v2.0.0/go.mod h1:YuVJ+D/dNt4HWrThTBnjgZuRbt7AuwINeg4q52ZE3Jw=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -47,21 +61,92 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8=
github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78=
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY=
github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY=
github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw=
github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -77,10 +162,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A=
github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@ -90,43 +171,70 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI=
github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -137,22 +245,39 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o=
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@ -176,10 +301,26 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw=
gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM=
gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to=
gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI=
gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc=
gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c=
gorm.io/driver/sqlserver v1.0.2/go.mod h1:gb0Y9QePGgqjzrVyTQUZeh9zkd5v0iz71cM1B4ZycEY=
gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.20.5 h1:g3tpSF9kggASzReK+Z3dYei1IJODLqNUbOjSuCczY8g=
gorm.io/gorm v1.20.5/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=

23
handlers.go
View File

@ -15,7 +15,6 @@ import (
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
const (
@ -98,7 +97,8 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request
key := vars["key"]
var p *db.Paste
var fu *db.FileUpload
err := rl.db.Bolt.View(func(tx *bolt.Tx) error {
var notFound bool
err := rl.db.Transaction(func(tx *db.Database) error {
var err error
p, err = db.GetPaste(tx, key)
if err != nil {
@ -114,6 +114,9 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request
}
return nil
})
if notFound {
err = db.ErrPasteDoesNotExist
}
if err != nil {
status := db.ErrHTTPStatusCode(err)
if status == http.StatusInternalServerError {
@ -122,7 +125,6 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request
rl.renderError(w, r, status, err.Error())
return
}
rl.viewPasteHandlerInner(w, r, flags, p, fu)
}
@ -282,7 +284,7 @@ func (rl *rushlink) newPasteHandler(w http.ResponseWriter, r *http.Request) {
func (rl *rushlink) newFileUploadPasteHandler(w http.ResponseWriter, r *http.Request, file multipart.File, header multipart.FileHeader) {
var fu *db.FileUpload
var paste *db.Paste
if err := rl.db.Bolt.Update(func(tx *bolt.Tx) error {
if err := rl.db.Transaction(func(tx *db.Database) error {
var err error
fu, err = db.NewFileUpload(rl.fs, file, header.Filename)
if err != nil {
@ -292,7 +294,7 @@ func (rl *rushlink) newFileUploadPasteHandler(w http.ResponseWriter, r *http.Req
panic(errors.Wrap(err, "saving fileUpload in db"))
}
paste, err = shortenFileUploadID(tx, fu.ID)
paste, err = shortenFileUploadID(tx, fu.PubID)
return err
}); err != nil {
panic(err)
@ -317,7 +319,7 @@ func (rl *rushlink) newRedirectPasteHandler(w http.ResponseWriter, r *http.Reque
}
var paste *db.Paste
if err := rl.db.Bolt.Update(func(tx *bolt.Tx) error {
if err := rl.db.Transaction(func(tx *db.Database) error {
var err error
paste, err = shortenURL(tx, userURL)
return err
@ -340,7 +342,7 @@ func (rl *rushlink) deletePasteHandler(w http.ResponseWriter, r *http.Request) {
var errorCode int
var paste *db.Paste
if err := rl.db.Bolt.Update(func(tx *bolt.Tx) error {
if err := rl.db.Transaction(func(tx *db.Database) error {
var err error
paste, err = db.GetPaste(tx, key)
if err != nil {
@ -372,19 +374,19 @@ func (rl *rushlink) deletePasteHandler(w http.ResponseWriter, r *http.Request) {
//
// Returns the new paste key if the fileUpload was successfully added to the
// database
func shortenFileUploadID(tx *bolt.Tx, id uuid.UUID) (*db.Paste, error) {
func shortenFileUploadID(tx *db.Database, id uuid.UUID) (*db.Paste, error) {
return shorten(tx, db.PasteTypeFileUpload, id[:])
}
// Add a new URL to the database
//
// Returns the new paste key if the url was successfully shortened
func shortenURL(tx *bolt.Tx, userURL *url.URL) (*db.Paste, error) {
func shortenURL(tx *db.Database, userURL *url.URL) (*db.Paste, error) {
return shorten(tx, db.PasteTypeRedirect, []byte(userURL.String()))
}
// Add a paste (of any kind) to the database with arbitrary content.
func shorten(tx *bolt.Tx, ty db.PasteType, content []byte) (*db.Paste, error) {
func shorten(tx *db.Database, ty db.PasteType, content []byte) (*db.Paste, error) {
// Generate the paste key
var keyEntropy int
if ty == db.PasteTypeFileUpload || ty == db.PasteTypePaste {
@ -408,7 +410,6 @@ func shorten(tx *bolt.Tx, ty db.PasteType, content []byte) (*db.Paste, error) {
Content: content,
Key: pasteKey,
DeleteToken: deleteToken,
TimeCreated: time.Now().UTC(),
}
if err := p.Save(tx); err != nil {
return nil, err

19
handlers_test.go
View File

@ -15,7 +15,6 @@ import (
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
"github.com/gorilla/mux"
"go.etcd.io/bbolt"
)
// createTemporaryRouter initializes a rushlink instance, with temporary
@ -35,16 +34,18 @@ func createTemporaryRouter(t *testing.T) (*mux.Router, *rushlink) {
if err != nil {
t.Fatalf("opening temporary filestore: %s\n", err)
}
databasePath := filepath.Join(tempDir, "rushlink.db")
database, err := db.OpenDB(databasePath, fileStore)
os.Setenv(db.EnvDatabaseDriver, "sqlite")
os.Setenv(db.EnvDatabasePath, "file::memory:?cache=shared")
database, err := db.OpenDBFromEnvironment()
if err != nil {
t.Fatalf("opening temporary database: %s\n", err)
t.Fatalf("opening temporary database: %v\n", err)
}
t.Cleanup(func() {
if err := database.Close(); err != nil {
t.Errorf("closing database: %d\n", err)
database.Debug()
if err := db.Gormigrate(database).Migrate(); err != nil {
t.Fatalf("migration failed: %v\n", err)
}
})
// *.invalid. is guaranteed not to exist (RFC 6761).
rootURL, err := url.Parse("https://rushlink.invalid")
@ -135,7 +136,7 @@ func TestIssue53(t *testing.T) {
checkStatusCode(t, rr, http.StatusFound)
// Check that any attempt to do directory traversal has failed.
rl.db.Bolt.View(func(tx *bbolt.Tx) error {
rl.db.Transaction(func(tx *db.Database) error {
fus, err := db.AllFileUploads(tx)
if err != nil {
t.Fatal(err)

204
internal/boltdb/db.go Normal file
View File

@ -0,0 +1,204 @@
package boltdb
import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"os"
"time"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
)
// Database is the main rushlink database type.
//
// Open a database using DB.Open() and close it in the end using DB.Close().
// Only one instance of DB should exist in a program at any moment.
type Database struct {
Bolt *bolt.DB
}
// CurrentMigrateVersion holds the current "migrate version".
//
// If we alter the database format, we bump this number and write a new
// database migration in migrate().
const CurrentMigrateVersion = 3
// BucketConf holds the name for the "configuration" bucket.
//
// This bucket holds the database version, secret site-wide keys, etc.
const BucketConf = "conf"
// BucketPastes holds the name for the pastes bucket.
const BucketPastes = "pastes"
// BucketFileUpload holds the name for the file-upload bucket.
const BucketFileUpload = "fileUpload"
// KeyMigrateVersion stores the current migration version. If this value is less than
// CurrentMigrateVersion, the database has to be migrated.
const KeyMigrateVersion = "migrate_version"
// OpenDB opens a database file located at path.
func OpenDB(path string, fs *FileStore) (*Database, error) {
if path == "" {
return nil, errors.New("database not set")
}
db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return nil, errors.Wrapf(err, "failed to open database at '%v'", path)
}
if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil {
return nil, err
}
return &Database{db}, nil
}
// Close the bolt database
func (db *Database) Close() error {
if db == nil {
panic("no open database")
}
return db.Bolt.Close()
}
// Initialize and migrate the database to the current version
func migrate(tx *bolt.Tx, fs *FileStore) error {
// Guidelines for error handling:
// - Errors based on malformed *structure* should be fatal!
// - Errors based on malformed *data* should print a warning
// (and if possible try to fix the error).
dbVersion, err := dbVersion(tx)
if err != nil {
return err
}
// Migrate the database to version 1
if dbVersion < 1 {
log.Println("migrating database to version 1")
// Create conf bucket
_, err := tx.CreateBucket([]byte(BucketConf))
if err != nil {
return err
}
// Create paste bucket
_, err = tx.CreateBucket([]byte(BucketPastes))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 1); err != nil {
return err
}
}
if dbVersion < 2 {
log.Println("migrating database to version 2")
// Create fileUpload bucket
_, err := tx.CreateBucket([]byte(BucketFileUpload))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 2); err != nil {
return err
}
}
if dbVersion < 3 {
log.Println("migrating database to version 3")
// In this version, we changed te way how Content-Types are being
// stored. Previously, we allowed clients to provide their own
// Content-Types for files, using the Content-Disposition header in
// multipart forms. The new way detects these types using
// http.DetectContentType.
//
// Scan through all the FileUploads and update their ContentTypes.
bucket := tx.Bucket([]byte(BucketFileUpload))
cursor := bucket.Cursor()
var id, storedBytes []byte
id, storedBytes = cursor.First()
for id != nil {
fu, err := decodeFileUpload(storedBytes)
if err != nil {
log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id))
id, storedBytes = cursor.Next()
continue
}
if fu.State != FileUploadStatePresent {
id, storedBytes = cursor.Next()
continue
}
filePath := fu.Path(fs)
file, err := os.Open(fu.Path(fs))
if err != nil {
log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath))
id, storedBytes = cursor.Next()
continue
}
var buf bytes.Buffer
buf.Grow(512)
io.CopyN(&buf, file, 512)
contentType := http.DetectContentType(buf.Bytes())
if contentType != fu.ContentType {
fu.ContentType = contentType
fu.Save(tx)
cursor.Seek(id)
}
id, storedBytes = cursor.Next()
}
// Update the version number
if err := setDBVersion(tx, 3); err != nil {
return err
}
}
return nil
}
// Get the current migrate version from the database
func dbVersion(tx *bolt.Tx) (int, error) {
conf := tx.Bucket([]byte(BucketConf))
if conf == nil {
return 0, nil
}
dbVersionBytes := conf.Get([]byte(KeyMigrateVersion))
if dbVersionBytes == nil {
return 0, nil
}
// Version was already stored
var dbVersion int
if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil {
return 0, err
}
if dbVersion == 0 {
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
}
if dbVersion > CurrentMigrateVersion {
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion)
}
return dbVersion, nil
}
// Update the current migrate version in the database
func setDBVersion(tx *bolt.Tx, version int) error {
conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf))
if err != nil {
return err
}
versionBytes, err := gobmarsh.Marshal(version)
if err != nil {
return err
}
return conf.Put([]byte(KeyMigrateVersion), versionBytes)
}

256
internal/boltdb/fileupload.go Normal file
View File

@ -0,0 +1,256 @@
package boltdb
import (
"bytes"
"encoding/hex"
"hash/crc32"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
)
// Use the Castagnoli checksum because of the acceleration on Intel CPUs
var checksumTable = crc32.MakeTable(crc32.Castagnoli)
// FileStore holds the path to a file storage location.
type FileStore struct {
path string
}
// FileUploadState determines the current state of a FileUpload object.
type FileUploadState int
// FileUpload models an uploaded file.
type FileUpload struct {
// State of the FileUpload (present/deleted/etc).
State FileUploadState
// ID identifies this FileUpload.
ID uuid.UUID
// FileName contains the original filename of this FileUpload.
FileName string
// Content type as determined by http.DetectContentType.
ContentType string
// Checksum holds a crc32c checksum of the file.
//
// This checksum is only meant to allow for the detection of random
// database corruption.
Checksum uint32
}
const (
dirMode os.FileMode = 0750
fileMode os.FileMode = 0640
)
const (
// FileUploadStateUndef is an undefined FileUpload.
FileUploadStateUndef FileUploadState = 0
// FileUploadStatePresent denotes the normal (existing) state.
FileUploadStatePresent FileUploadState = 1
// FileUploadStateDeleted denotes a deleted state.
FileUploadStateDeleted FileUploadState = 2
)
func (t FileUploadState) String() string {
switch t {
case FileUploadStateUndef:
return "unknown"
case FileUploadStatePresent:
return "present"
case FileUploadStateDeleted:
return "deleted"
default:
return "invalid"
}
}
// OpenFileStore opens the file storage at path.
func OpenFileStore(path string) (*FileStore, error) {
if path == "" {
return nil, errors.New("file-store not set")
}
// Try to create the file store directory if it does not yet exist
if err := os.MkdirAll(path, dirMode); err != nil {
return nil, errors.Wrap(err, "creating file store directory")
}
return &FileStore{path[:]}, nil
}
// Path returns the path of the FileStore root.
func (fs *FileStore) Path() string {
return fs.path
}
// filePath resolves the path of a file in the FileStore given some id and filename.
func (fs *FileStore) filePath(id uuid.UUID, fileName string) string {
if fs.path == "" {
panic("fileStoreDir called while the file store path has not been set")
}
return path.Join(fs.path, hex.EncodeToString(id[:]), fileName)
}
// NewFileUpload creates a new FileUpload object.
//
// Internally, this function detects the type of the file stored in `r` using
// `http.DetectContentType`.
func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) {
// Generate a file ID
id, err := uuid.NewRandom()
if err != nil {
return nil, errors.Wrap(err, "generating UUID")
}
// Construct a checksum for this file
hash := crc32.New(checksumTable)
tee := io.TeeReader(r, hash)
// Detect the file type
var tmpBuf bytes.Buffer
tmpBuf.Grow(512)
io.CopyN(&tmpBuf, tee, 512)
contentType := http.DetectContentType(tmpBuf.Bytes())
// Open the file on disk for writing
baseName := filepath.Base(fileName)
filePath := fs.filePath(id, baseName)
if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil {
return nil, errors.Wrap(err, "creating file dir")
}
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode)
if err != nil {
return nil, errors.Wrap(err, "opening file")
}
defer file.Close()
// Write the file to disk
_, err = io.Copy(file, &tmpBuf)
if err != nil {
return nil, errors.Wrap(err, "writing to file")
}
_, err = io.Copy(file, tee)
if err != nil {
return nil, errors.Wrap(err, "writing to file")
}
fu := &FileUpload{
State: FileUploadStatePresent,
ID: id,
FileName: baseName,
ContentType: contentType,
Checksum: hash.Sum32(),
}
return fu, nil
}
// GetFileUpload tries to retrieve a FileUpload object from the bolt database.
func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
storedBytes := bucket.Get(id[:])
if storedBytes == nil {
return nil, nil
}
return decodeFileUpload(storedBytes)
}
// AllFileUploads tries to retrieve all FileUpload objects from the bolt database.
func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
var fus []FileUpload
err := bucket.ForEach(func(_, storedBytes []byte) error {
fu, err := decodeFileUpload(storedBytes)
if err != nil {
return err
}
fus = append(fus, *fu)
return nil
})
if err != nil {
return nil, err
}
return fus, nil
}
func decodeFileUpload(storedBytes []byte) (*FileUpload, error) {
fu := &FileUpload{}
err := gobmarsh.Unmarshal(storedBytes, fu)
return fu, err
}
// Save saves a FileUpload in the database.
func (fu *FileUpload) Save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
buf, err := gobmarsh.Marshal(fu)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put(fu.ID[:], buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
}
// Delete deletes a FileUpload from the database.
func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
// Remove the file in the backend
filePath := fu.Path(fs)
if err := os.Remove(filePath); err != nil {
return err
}
// Update the file in the server
if err := (&FileUpload{
ID: fu.ID,
State: FileUploadStateDeleted,
}).Save(tx); err != nil {
return err
}
// Cleanup the parent directory
wrap := "deletion succeeded, but removing the file directory has failed"
return errors.Wrap(os.Remove(path.Dir(filePath)), wrap)
}
// Path returns the path to this FileUpload in the FileStore provided in fs.
func (fu *FileUpload) Path(fs *FileStore) string {
return fs.filePath(fu.ID, fu.FileName)
}
// URL returns the URL for the FileUpload.
func (fu *FileUpload) URL() *url.URL {
rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName
urlParse, err := url.Parse(rawurl)
if err != nil {
panic("could not construct /uploads/ url")
}
return urlParse
}
// Ext returns the extension of the file attached to this FileUpload.
func (fu *FileUpload) Ext() string {
return filepath.Ext(fu.FileName)
}

345
internal/boltdb/paste.go Normal file
View File

@ -0,0 +1,345 @@
package boltdb
import (
"crypto/rand"
"encoding/base64"
"encoding/hex"
"net/http"
"net/url"
"strings"
"time"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
// PasteType describes the type of Paste (i.e. file, redirect, [...]).
type PasteType int
// PasteState describes the state of a Paste (i.e. present, deleted, [...]).
type PasteState int
// Paste describes the main Paste model in the database.
type Paste struct {
Type PasteType
State PasteState
Content []byte
Key string
DeleteToken string
TimeCreated time.Time
}
// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen
var ReservedPasteKeys = []string{"xd42", "example"}
// Note: we use iota here. That means removals of PasteType* are not allowed,
// because this changes the value of the constant. Please add the comment
// "// deprecated" if you want to remove the constant. Additions are only
// allowed at the bottom of this block, for the same reason.
const (
PasteTypeUndef PasteType = iota
// PasteTypePaste is as of yet unused. It is still unclear if this type
// will ever get a proper meaning.
PasteTypePaste
PasteTypeRedirect
PasteTypeFileUpload
)
// Note: we use iota here. See the comment above PasteType*
const (
PasteStateUndef PasteState = iota
PasteStatePresent
PasteStateDeleted
)
// minKeyLen specifies the mimimum length of a paste key.
const minKeyLen = 4
var (
// ErrKeyInvalidChar occurs when a key contains an invalid character.
ErrKeyInvalidChar = errors.New("invalid character in key")
// ErrKeyInvalidLength occurs when a key embeds a length that is incorrect.
ErrKeyInvalidLength = errors.New("key length encoding is incorrect")
// ErrPasteDoesNotExist occurs when a key does not exist in the database.
ErrPasteDoesNotExist = errors.New("url key not found in the database")
)
// ErrHTTPStatusCode returns the HTTP status code that should correspond to
// the provided error.
// server error, or false if it is not.
func ErrHTTPStatusCode(err error) int {
switch err {
case nil:
return 0
case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist:
return http.StatusNotFound
}
return http.StatusInternalServerError
}
// Base64 encoding and decoding
var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding)
func (t PasteType) String() string {
switch t {
case PasteTypeUndef:
return "unknown"
case PasteTypePaste:
return "paste"
case PasteTypeRedirect:
return "redirect"
case PasteTypeFileUpload:
return "file"
default:
return "invalid"
}
}
func (t PasteState) String() string {
switch t {
case PasteStateUndef:
return "unknown"
case PasteStatePresent:
return "present"
case PasteStateDeleted:
return "deleted"
default:
return "invalid"
}
}
// GetPaste retrieves a paste from the database.
func GetPaste(tx *bolt.Tx, key string) (*Paste, error) {
if err := ValidatePasteKey(key); err != nil {
return nil, err
}
return GetPasteNoValidate(tx, key)
}
// ValidatePasteKey validates the format of the key that has
func ValidatePasteKey(key string) error {
internalLen := minKeyLen
countingOnes := true
for _, ch := range key {
limb := strings.IndexRune(base64Alphabet, ch)
if limb == -1 {
return ErrKeyInvalidChar
}
for i := 5; i >= 0 && countingOnes; i-- {
if (limb>>uint(i))&0x1 == 0 {
countingOnes = false
break
}
internalLen++
}
}
if internalLen != len(key) {
return ErrKeyInvalidLength
}
return nil
}
// GetPasteNoValidate retrieves a paste from the database without validating
// the key format first.
func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) {
pastesBucket := tx.Bucket([]byte(BucketPastes))
if pastesBucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
}
storedBytes := pastesBucket.Get([]byte(key))
if storedBytes == nil {
return nil, ErrPasteDoesNotExist
}
return decodePaste(storedBytes)
}
func decodePaste(storedBytes []byte) (*Paste, error) {
p := &Paste{}
err := gobmarsh.Unmarshal(storedBytes, p)
return p, err
}
// Save saves this Paste to the database.
func (p *Paste) Save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketPastes))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BucketPastes)
}
buf, err := gobmarsh.Marshal(p)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put([]byte(p.Key), buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
}
// Delete deletes this Paste from the database.
func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error {
// Remove the (maybe) attached file
if p.Type == PasteTypeFileUpload {
fuID, err := uuid.FromBytes(p.Content)
if err != nil {
return errors.Wrap(err, "failed to parse uuid")
}
fu, err := GetFileUpload(tx, fuID)
if err != nil {
return errors.Wrap(err, "failed to find file in database")
}
if err := fu.Delete(tx, fs); err != nil {
return errors.Wrap(err, "failed to remove file")
}
}
// Replace the old paste with a new empty paste
p.Type = PasteTypeUndef
p.State = PasteStateDeleted
p.Content = []byte{}
if err := p.Save(tx); err != nil {
return errors.Wrap(err, "failed to delete paste in database")
}
return nil
}
// RedirectURL returns the URL from this paste.
//
// This function assumes that the paste is valid. If the paste struct is
// corrupted in some way, this function will panic.
func (p *Paste) RedirectURL() *url.URL {
if p.Type != PasteTypeRedirect {
panic("expected p.Type to be PasteTypeRedirect")
}
rawurl := string(p.Content)
urlParse, err := url.Parse(rawurl)
if err != nil {
panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key))
}
return urlParse
}
// GeneratePasteKey generates a new paste key. It will ensure that the newly
// generated paste key does not already exist in the database.
// The running time of this function is in O(log N), where N is the amount of
// keys stored in the url-shorten database.
// In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum
// guessing entropy of the generated key.
func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) {
pastesBucket := tx.Bucket([]byte(BucketPastes))
if pastesBucket == nil {
return "", errors.Errorf("bucket %v does not exist", BucketPastes)
}
epoch := 0
var key string
for {
var err error
key, err = generatePasteKeyInner(epoch, minimumEntropy)
if err != nil {
return "", errors.Wrap(err, "url-key generation failed")
}
found := pastesBucket.Get([]byte(key))
if found == nil {
break
}
isReserved := false
for _, reservedKey := range ReservedPasteKeys {
if strings.HasPrefix(key, reservedKey) {
isReserved = true
break
}
}
if !isReserved {
break
}
epoch++
}
return key, nil
}
// generatePasteKeyInner generates a new paste key, but leaves the
// uniqueness and is-reserved checks to the caller. That is, it only
// generates a random key in the correct (syntactical) format.
// Both epoch and entropy can be used to set the key length. Epoch is used
// to prevent collisions in retrying to generate new keys. Entropy (in bits)
// is used to ensure that a new key has at least some amount of guessing
// entropy.
func generatePasteKeyInner(epoch, entropy int) (string, error) {
entropyEpoch := entropy
entropyEpoch -= minKeyLen * 6 // First 4 characters provide 24 bits.
entropyEpoch++ // One bit less because of '0' bit.
entropyEpoch = (entropyEpoch-1)/5 + 1 // 5 bits for every added epoch.
if epoch < entropyEpoch {
epoch = entropyEpoch
}
urlKey := make([]byte, minKeyLen+epoch)
_, err := rand.Read(urlKey)
if err != nil {
return "", err
}
// Put all the values in the range 0..64 for easier base64-encoding
for i := 0; i < len(urlKey); i++ {
urlKey[i] &= 0x3F
}
// Implement truncate-resistance by forcing the prefix to
// 0b111110xxxxxxxxxx
// ^----- {epoch} ones followed by a single 0
//
// Example when epoch is 1: prefix is 0b10.
i := 0
for i < epoch {
// Set this bit to 1
limb := i / 6
bit := i % 6
urlKey[limb] |= 1 << uint(5-bit)
i++
}
// Finally set the next bit to 0
limb := i / 6
bit := i % 6
urlKey[limb] &= ^(1 << uint(5-bit))
// Convert this ID to a canonical base64 notation
for i := range urlKey {
urlKey[i] = base64Alphabet[urlKey[i]]
}
return string(urlKey), nil
}
// GenerateDeleteToken generates a new (random) delete token.
func GenerateDeleteToken() (string, error) {
var deleteToken [16]byte
_, err := rand.Read(deleteToken[:])
if err != nil {
return "", err
}
return hex.EncodeToString(deleteToken[:]), nil
}
// AllPastes tries to retrieve all the Paste objects from the database.
func AllPastes(tx *bolt.Tx) ([]Paste, error) {
bucket := tx.Bucket([]byte(BucketPastes))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
}
var ps []Paste
err := bucket.ForEach(func(_, storedBytes []byte) error {
p, err := decodePaste(storedBytes)
if err != nil {
return err
}
ps = append(ps, *p)
return nil
})
if err != nil {
return nil, err
}
return ps, nil
}

38
internal/boltdb/paste_test.go Normal file
View File

@ -0,0 +1,38 @@
package boltdb
import "testing"
func TestValidatePasteKey(t *testing.T) {
tests := []struct {
key string
wantErr bool
errKind error
}{
{"xd42__", false, nil},
{"xd42_*", true, ErrKeyInvalidChar},
{"xd42_/", true, ErrKeyInvalidChar},
{"xd42_=", true, ErrKeyInvalidChar},
{"xd42_", true, ErrKeyInvalidLength},
{"xd42", true, ErrKeyInvalidLength},
{"xd4", true, ErrKeyInvalidLength},
{"xd", true, ErrKeyInvalidLength},
{"x", true, ErrKeyInvalidLength},
{"", true, ErrKeyInvalidLength},
{"KoJ5", false, nil},
{"__dGSJIIbBpr-SD0", false, nil},
{"__dGSJIIbBpr-SD", true, ErrKeyInvalidLength},
}
for _, tt := range tests {
t.Run(tt.key, func(t *testing.T) {
err := ValidatePasteKey(tt.key)
if (err != nil) != tt.wantErr {
t.Errorf("ValidatePasteKey() got error = %v, want error %v", err != nil, tt.wantErr)
}
if (err != nil) && err != tt.errKind {
t.Errorf("ValidatePasteKey() error = %v, want errKind %v", err, tt.errKind)
}
})
}
}

234
internal/db/db.go
View File

@ -1,204 +1,74 @@
package db
import (
"bytes"
"fmt"
"io"
"log"
"net/http"
"os"
"time"
"github.com/pkg/errors"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
bolt "go.etcd.io/bbolt"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
const (
// EnvDatabaseDriver is the key for the 'database driver' environment variable.
EnvDatabaseDriver = "RUSHLINK_DATABASE_DRIVER"
// EnvDatabasePath is the key for the 'database path' environment variable.
EnvDatabasePath = "RUSHLINK_DATABASE_PATH"
// EnvPostgresURL is the key for the 'postgresql URL' environment variable.
EnvPostgresURL = "RUSHLINK_POSTGRES_URL"
// EnvFileStorePath is the key for the environment variable locating the file store.
EnvFileStorePath = "RUSHLINK_FILE_STORE_PATH"
)
// Database is the main rushlink database type.
//
// Open a database using DB.Open() and close it in the end using DB.Close().
// Open a database using OpenDBFromEnvironment(). Closing is not necessary.
// Only one instance of DB should exist in a program at any moment.
type Database struct {
Bolt *bolt.DB
}
type Database = gorm.DB
// CurrentMigrateVersion holds the current "migrate version".
//
// If we alter the database format, we bump this number and write a new
// database migration in migrate().
const CurrentMigrateVersion = 3
var (
gormLogger logger.Interface = logger.New(log.New(os.Stdout, "\r\n", log.LstdFlags), logger.Config{
SlowThreshold: 50 * time.Millisecond,
LogLevel: logger.Warn,
Colorful: true,
})
gormConfig = gorm.Config{Logger: gormLogger, PrepareStmt: true}
)
// BucketConf holds the name for the "configuration" bucket.
//
// This bucket holds the database version, secret site-wide keys, etc.
const BucketConf = "conf"
// OpenDBFromEnvironment tries to open an SQL database, described by
func OpenDBFromEnvironment() (*Database, error) {
const envNotSetMsg = "%v environment variable is not set"
// BucketPastes holds the name for the pastes bucket.
const BucketPastes = "pastes"
// BucketFileUpload holds the name for the file-upload bucket.
const BucketFileUpload = "fileUpload"
// KeyMigrateVersion stores the current migration version. If this value is less than
// CurrentMigrateVersion, the database has to be migrated.
const KeyMigrateVersion = "migrate_version"
// OpenDB opens a database file located at path.
func OpenDB(path string, fs *FileStore) (*Database, error) {
if path == "" {
return nil, errors.New("database not set")
driver, prs := os.LookupEnv(EnvDatabaseDriver)
if !prs {
return nil, errors.Errorf(envNotSetMsg, EnvDatabaseDriver)
}
db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second})
switch driver {
case "sqlite":
path, prs := os.LookupEnv(EnvDatabasePath)
if !prs {
return nil, errors.Errorf(envNotSetMsg, EnvDatabasePath)
}
db, err := gorm.Open(sqlite.Open(path), &gormConfig)
if err != nil {
return nil, errors.Wrapf(err, "failed to open database at '%v'", path)
}
if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil {
return nil, err
}
return &Database{db}, nil
}
// Close the bolt database
func (db *Database) Close() error {
if db == nil {
panic("no open database")
}
return db.Bolt.Close()
}
// Initialize and migrate the database to the current version
func migrate(tx *bolt.Tx, fs *FileStore) error {
// Guidelines for error handling:
// - Errors based on malformed *structure* should be fatal!
// - Errors based on malformed *data* should print a warning
// (and if possible try to fix the error).
dbVersion, err := dbVersion(tx)
if err != nil {
return err
}
// Migrate the database to version 1
if dbVersion < 1 {
log.Println("migrating database to version 1")
// Create conf bucket
_, err := tx.CreateBucket([]byte(BucketConf))
if err != nil {
return err
}
// Create paste bucket
_, err = tx.CreateBucket([]byte(BucketPastes))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 1); err != nil {
return err
}
}
if dbVersion < 2 {
log.Println("migrating database to version 2")
// Create fileUpload bucket
_, err := tx.CreateBucket([]byte(BucketFileUpload))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 2); err != nil {
return err
}
}
if dbVersion < 3 {
log.Println("migrating database to version 3")
// In this version, we changed te way how Content-Types are being
// stored. Previously, we allowed clients to provide their own
// Content-Types for files, using the Content-Disposition header in
// multipart forms. The new way detects these types using
// http.DetectContentType.
//
// Scan through all the FileUploads and update their ContentTypes.
bucket := tx.Bucket([]byte(BucketFileUpload))
cursor := bucket.Cursor()
var id, storedBytes []byte
id, storedBytes = cursor.First()
for id != nil {
fu, err := decodeFileUpload(storedBytes)
if err != nil {
log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id))
id, storedBytes = cursor.Next()
continue
}
if fu.State != FileUploadStatePresent {
id, storedBytes = cursor.Next()
continue
}
filePath := fu.Path(fs)
file, err := os.Open(fu.Path(fs))
if err != nil {
log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath))
id, storedBytes = cursor.Next()
continue
}
var buf bytes.Buffer
buf.Grow(512)
io.CopyN(&buf, file, 512)
contentType := http.DetectContentType(buf.Bytes())
if contentType != fu.ContentType {
fu.ContentType = contentType
fu.Save(tx)
cursor.Seek(id)
}
id, storedBytes = cursor.Next()
}
// Update the version number
if err := setDBVersion(tx, 3); err != nil {
return err
}
}
return nil
}
// Get the current migrate version from the database
func dbVersion(tx *bolt.Tx) (int, error) {
conf := tx.Bucket([]byte(BucketConf))
if conf == nil {
return 0, nil
}
dbVersionBytes := conf.Get([]byte(KeyMigrateVersion))
if dbVersionBytes == nil {
return 0, nil
}
// Version was already stored
var dbVersion int
if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil {
return 0, err
}
if dbVersion == 0 {
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
}
if dbVersion > CurrentMigrateVersion {
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion)
}
return dbVersion, nil
}
// Update the current migrate version in the database
func setDBVersion(tx *bolt.Tx, version int) error {
conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf))
if err != nil {
return err
}
versionBytes, err := gobmarsh.Marshal(version)
if err != nil {
return err
}
return conf.Put([]byte(KeyMigrateVersion), versionBytes)
return db, nil
case "postgres":
dsn, prs := os.LookupEnv(EnvPostgresURL)
if !prs {
return nil, errors.Errorf(envNotSetMsg, EnvPostgresURL)
}
db, err := gorm.Open(postgres.Open(dsn), &gormConfig)
if err != nil {
return nil, err
}
return db, nil
default:
return nil, errors.Errorf("RUSHLINK_DATABASE_DRIVER should be either 'sqlite' or 'postgres' (not '%v'), "+
"for more info see <https://stackoverflow.com/q/3582552/5207081>", driver)
}
}

106
internal/db/fileupload.go
View File

@ -10,12 +10,11 @@ import (
"os"
"path"
"path/filepath"
"time"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
"gorm.io/gorm"
)
// Use the Castagnoli checksum because of the acceleration on Intel CPUs
@ -31,11 +30,13 @@ type FileUploadState int
// FileUpload models an uploaded file.
type FileUpload struct {
// State of the FileUpload (present/deleted/etc).
State FileUploadState
ID uint `gorm:"primaryKey"`
// ID identifies this FileUpload.
ID uuid.UUID
// State of the FileUpload (present/deleted/etc).
State FileUploadState `gorm:"index"`
// UUID publically identifies this FileUpload.
PubID uuid.UUID `gorm:"uniqueIndex"`
// FileName contains the original filename of this FileUpload.
FileName string
@ -48,6 +49,10 @@ type FileUpload struct {
// This checksum is only meant to allow for the detection of random
// database corruption.
Checksum uint32
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt
}
const (
@ -77,6 +82,19 @@ func (t FileUploadState) String() string {
}
}
// ErrFileUploadDoesNotExist occurs when a key does not exist in the database.
var ErrFileUploadDoesNotExist = errors.New("file not found in the database")
// OpenFileStoreFromEnvironment tries to open a file store located at ${RUSHLINK_FILE_STORE_PATH}.
func OpenFileStoreFromEnvironment() (*FileStore, error) {
path, prs := os.LookupEnv(EnvFileStorePath)
if !prs {
err := errors.Errorf("%v environment variable is not set", EnvFileStorePath)
return nil, errors.Wrap(err, "opening file store")
}
return OpenFileStore(path)
}
// OpenFileStore opens the file storage at path.
func OpenFileStore(path string) (*FileStore, error) {
if path == "" {
@ -97,11 +115,11 @@ func (fs *FileStore) Path() string {
}
// filePath resolves the path of a file in the FileStore given some id and filename.
func (fs *FileStore) filePath(id uuid.UUID, fileName string) string {
func (fs *FileStore) filePath(pubID uuid.UUID, fileName string) string {
if fs.path == "" {
panic("fileStoreDir called while the file store path has not been set")
}
return path.Join(fs.path, hex.EncodeToString(id[:]), fileName)
return path.Join(fs.path, hex.EncodeToString(pubID[:]), fileName)
}
// NewFileUpload creates a new FileUpload object.
@ -110,7 +128,7 @@ func (fs *FileStore) filePath(id uuid.UUID, fileName string) string {
// `http.DetectContentType`.
func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) {
// Generate a file ID
id, err := uuid.NewRandom()
pubID, err := uuid.NewRandom()
if err != nil {
return nil, errors.Wrap(err, "generating UUID")
}
@ -127,7 +145,7 @@ func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, er
// Open the file on disk for writing
baseName := filepath.Base(fileName)
filePath := fs.filePath(id, baseName)
filePath := fs.filePath(pubID, baseName)
if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil {
return nil, errors.Wrap(err, "creating file dir")
}
@ -149,7 +167,7 @@ func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, er
fu := &FileUpload{
State: FileUploadStatePresent,
ID: id,
PubID: pubID,
FileName: baseName,
ContentType: contentType,
Checksum: hash.Sum32(),
@ -158,64 +176,33 @@ func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, er
}
// GetFileUpload tries to retrieve a FileUpload object from the bolt database.
func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
func GetFileUpload(db *gorm.DB, pubID uuid.UUID) (*FileUpload, error) {
var fus []FileUpload
if err := db.Unscoped().Limit(1).Where("pub_id = ?", pubID).Find(&fus).Error; err != nil {
return nil, err
}
storedBytes := bucket.Get(id[:])
if storedBytes == nil {
return nil, nil
if len(fus) == 0 {
return nil, ErrFileUploadDoesNotExist
}
return decodeFileUpload(storedBytes)
return &fus[0], nil
}
// AllFileUploads tries to retrieve all FileUpload objects from the bolt database.
func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
func AllFileUploads(db *gorm.DB) ([]FileUpload, error) {
var fus []FileUpload
err := bucket.ForEach(func(_, storedBytes []byte) error {
fu, err := decodeFileUpload(storedBytes)
if err != nil {
return err
}
fus = append(fus, *fu)
return nil
})
if err != nil {
if err := db.Find(&fus).Error; err != nil {
return nil, err
}
return fus, nil
}
func decodeFileUpload(storedBytes []byte) (*FileUpload, error) {
fu := &FileUpload{}
err := gobmarsh.Unmarshal(storedBytes, fu)
return fu, err
}
// Save saves a FileUpload in the database.
func (fu *FileUpload) Save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketFileUpload))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BucketFileUpload)
}
buf, err := gobmarsh.Marshal(fu)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put(fu.ID[:], buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
func (fu *FileUpload) Save(db *gorm.DB) error {
return db.Save(fu).Error
}
// Delete deletes a FileUpload from the database.
func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
func (fu *FileUpload) Delete(db *gorm.DB, fs *FileStore) error {
// Remove the file in the backend
filePath := fu.Path(fs)
if err := os.Remove(filePath); err != nil {
@ -223,10 +210,7 @@ func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
}
// Update the file in the server
if err := (&FileUpload{
ID: fu.ID,
State: FileUploadStateDeleted,
}).Save(tx); err != nil {
if err := db.Delete(fu).Error; err != nil {
return err
}
@ -237,12 +221,12 @@ func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error {
// Path returns the path to this FileUpload in the FileStore provided in fs.
func (fu *FileUpload) Path(fs *FileStore) string {
return fs.filePath(fu.ID, fu.FileName)
return fs.filePath(fu.PubID, fu.FileName)
}
// URL returns the URL for the FileUpload.
func (fu *FileUpload) URL() *url.URL {
rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName
rawurl := "/uploads/" + hex.EncodeToString(fu.PubID[:]) + "/" + fu.FileName
urlParse, err := url.Parse(rawurl)
if err != nil {
panic("could not construct /uploads/ url")

46
internal/db/migrate.go Normal file
View File

@ -0,0 +1,46 @@
package db
import (
"time"
gormigrate "github.com/go-gormigrate/gormigrate/v2"
"github.com/google/uuid"
"gorm.io/gorm"
)
// Gormigrate returns a Gormigrate migrator for the database.
func Gormigrate(db *gorm.DB) *gormigrate.Gormigrate {
return gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{
{
ID: "202010251337",
Migrate: func(tx *gorm.DB) error {
type FileUpload struct {
ID uint `gorm:"primaryKey"`
State FileUploadState `gorm:"index"`
PubID uuid.UUID `gorm:"uniqueIndex"`
FileName string
ContentType string
Checksum uint32
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt
}
type Paste struct {
ID uint `gorm:"primaryKey"`
Type PasteType `gorm:"index"`
State PasteState `gorm:"index"`
Content []byte
Key string `gorm:"uniqueIndex"`
DeleteToken string
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt
}
return tx.AutoMigrate(&FileUpload{}, &Paste{})
},
Rollback: func(tx *gorm.DB) error {
return tx.Migrator().DropTable(&FileUpload{}, &Paste{})
},
},
})
}

82
internal/db/paste.go
View File

@ -12,7 +12,7 @@ import (
gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
"gorm.io/gorm"
)
// PasteType describes the type of Paste (i.e. file, redirect, [...]).
@ -23,15 +23,18 @@ type PasteState int
// Paste describes the main Paste model in the database.
type Paste struct {
Type PasteType
State PasteState
ID uint `gorm:"primaryKey"`
Type PasteType `gorm:"index"`
State PasteState `gorm:"index"`
Content []byte
Key string
Key string `gorm:"uniqueIndex"`
DeleteToken string
TimeCreated time.Time
CreatedAt time.Time
UpdatedAt time.Time
DeletedAt gorm.DeletedAt
}
// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen
// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen.
var ReservedPasteKeys = []string{"xd42", "example"}
// Note: we use iota here. That means removals of PasteType* are not allowed,
@ -73,7 +76,7 @@ func ErrHTTPStatusCode(err error) int {
switch err {
case nil:
return 0
case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist:
case gorm.ErrRecordNotFound, ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist:
return http.StatusNotFound
}
return http.StatusInternalServerError
@ -112,11 +115,11 @@ func (t PasteState) String() string {
}
// GetPaste retrieves a paste from the database.
func GetPaste(tx *bolt.Tx, key string) (*Paste, error) {
func GetPaste(db *gorm.DB, key string) (*Paste, error) {
if err := ValidatePasteKey(key); err != nil {
return nil, err
}
return GetPasteNoValidate(tx, key)
return GetPasteNoValidate(db, key)
}
// ValidatePasteKey validates the format of the key that has
@ -144,16 +147,15 @@ func ValidatePasteKey(key string) error {
// GetPasteNoValidate retrieves a paste from the database without validating
// the key format first.
func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) {
pastesBucket := tx.Bucket([]byte(BucketPastes))
if pastesBucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BucketPastes)
func GetPasteNoValidate(db *gorm.DB, key string) (*Paste, error) {
var ps []Paste
if err := db.Unscoped().Limit(1).Where("key = ?", key).Find(&ps).Error; err != nil {
return nil, err
}
storedBytes := pastesBucket.Get([]byte(key))
if storedBytes == nil {
if len(ps) == 0 {
return nil, ErrPasteDoesNotExist
}
return decodePaste(storedBytes)
return &ps[0], nil
}
func decodePaste(storedBytes []byte) (*Paste, error) {
@ -163,44 +165,36 @@ func decodePaste(storedBytes []byte) (*Paste, error) {
}
// Save saves this Paste to the database.
func (p *Paste) Save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BucketPastes))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BucketPastes)
}
buf, err := gobmarsh.Marshal(p)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put([]byte(p.Key), buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
func (p *Paste) Save(db *gorm.DB) error {
return db.Save(p).Error
}
// Delete deletes this Paste from the database.
func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error {
func (p *Paste) Delete(db *gorm.DB, fs *FileStore) error {
// Remove the (maybe) attached file
if p.Type == PasteTypeFileUpload {
fuID, err := uuid.FromBytes(p.Content)
if err != nil {
return errors.Wrap(err, "failed to parse uuid")
}
fu, err := GetFileUpload(tx, fuID)
fu, err := GetFileUpload(db, fuID)
if err != nil {
return errors.Wrap(err, "failed to find file in database")
}
if err := fu.Delete(tx, fs); err != nil {
if err := fu.Delete(db, fs); err != nil {
return errors.Wrap(err, "failed to remove file")
}
}
// Replace the old paste with a new empty paste
// Wipe the old paste
p.Type = PasteTypeUndef
p.State = PasteStateDeleted
p.Content = []byte{}
if err := p.Save(tx); err != nil {
if err := db.Save(&p).Error; err != nil {
return errors.Wrap(err, "failed to wipe paste in database")
}
// Soft-delete the paste as well
if err := db.Delete(&p).Error; err != nil {
return errors.Wrap(err, "failed to delete paste in database")
}
return nil
@ -228,12 +222,7 @@ func (p *Paste) RedirectURL() *url.URL {
// keys stored in the url-shorten database.
// In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum
// guessing entropy of the generated key.
func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) {
pastesBucket := tx.Bucket([]byte(BucketPastes))
if pastesBucket == nil {
return "", errors.Errorf("bucket %v does not exist", BucketPastes)
}
func GeneratePasteKey(db *gorm.DB, minimumEntropy int) (string, error) {
epoch := 0
var key string
for {
@ -243,10 +232,12 @@ func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) {
return "", errors.Wrap(err, "url-key generation failed")
}
found := pastesBucket.Get([]byte(key))
if found == nil {
break
var count int64
db.Unscoped().Model(&Paste{}).Where("key = ?", []byte(key)).Count(&count)
if err != nil {
return "", errors.Wrap(err, "failed to check if key already exists")
}
alreadyInUse := count != 0
isReserved := false
for _, reservedKey := range ReservedPasteKeys {
@ -255,10 +246,9 @@ func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) {
break
}
}
if !isReserved {
if !alreadyInUse && !isReserved {
break
}
epoch++
}
return key, nil

64
internal/db/paste_test.go
View File

@ -1,6 +1,24 @@
package db
import "testing"
import (
"testing"
"testing/quick"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func OpenTemporaryDB() (*Database, error) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gormConfig)
if err != nil {
return nil, err
}
migrate := Gormigrate(db)
if err := migrate.Migrate(); err != nil {
return nil, err
}
return db, nil
}
func TestValidatePasteKey(t *testing.T) {
tests := []struct {
@ -36,3 +54,47 @@ func TestValidatePasteKey(t *testing.T) {
})
}
}
func TestGeneratedKeysAreValid(t *testing.T) {
db, err := OpenTemporaryDB()
if err != nil {
t.Error(err)
}
var minimumEntropy int
checkGeneratedKeyValid := func() bool {
key, err := GeneratePasteKey(db, minimumEntropy)
if err != nil {
return false
}
return ValidatePasteKey(key) == nil
}
for minimumEntropy = 0; minimumEntropy <= 80; minimumEntropy++ {
if err = quick.Check(checkGeneratedKeyValid, nil); err != nil {
t.Error(err)
}
}
}
func TestTruncatedKeysAreInvalid(t *testing.T) {
db, err := OpenTemporaryDB()
if err != nil {
t.Error(err)
}
var minimumEntropy int
checkTruncatedKeysInvalid := func(truncRand uint) bool {
key, err := GeneratePasteKey(db, minimumEntropy)
if err != nil {
return false
}
trunc := int(truncRand % uint(len(key)-1))
key = key[:trunc]
return ValidatePasteKey(key) == ErrKeyInvalidLength
}
for minimumEntropy = 0; minimumEntropy <= 80; minimumEntropy++ {
if err = quick.Check(checkTruncatedKeysInvalid, nil); err != nil {
t.Error(err)
}
}
}

12
metrics.go
View File

@ -6,12 +6,10 @@ import (
"time"
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
bolt "go.etcd.io/bbolt"
)
const metricNamespace = "rushlink"
@ -25,12 +23,12 @@ var metricRequestsTotalCounter = prometheus.NewCounterVec(prometheus.CounterOpts
func metricURLsTotal(database *db.Database) float64 {
var metric float64
if err := database.Bolt.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte("pastes"))
if bucket == nil {
return errors.New("bucket 'pastes' could not be found")
if err := database.Transaction(func(tx *db.Database) error {
var count int64
if err := database.Model(&db.Paste{}).Count(&count).Error; err != nil {
return err
}
metric = float64(bucket.Stats().KeyN)
metric = float64(count)
return nil
}); err != nil {
log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric"))