diff --git a/assets/templates/html/pasteMeta.html.tmpl b/assets/templates/html/pasteMeta.html.tmpl index dbbf7bc..9056290 100644 --- a/assets/templates/html/pasteMeta.html.tmpl +++ b/assets/templates/html/pasteMeta.html.tmpl @@ -20,10 +20,18 @@ with delete token: <unknown> {{end -}} type: {{.Paste.Type}} state: {{.Paste.State}} -{{if .Paste.TimeCreated.IsZero -}} +{{if .Paste.CreatedAt.IsZero -}} created: unknown {{else -}} -created: {{.Paste.TimeCreated}} +created: {{.Paste.CreatedAt}} +{{end -}} +{{if .Paste.UpdatedAt.IsZero -}} +last updated: unknown +{{else -}} +last updated: {{.Paste.UpdatedAt}} +{{end -}} +{{if .Paste.DeletedAt.Valid -}} +deleted at: {{.Paste.DeletedAt.Time}} {{end -}} delete token: {{.CanDeleteString}}
diff --git a/assets/templates/txt/pasteMeta.txt.tmpl b/assets/templates/txt/pasteMeta.txt.tmpl index 84aeef5..a0b4021 100644 --- a/assets/templates/txt/pasteMeta.txt.tmpl +++ b/assets/templates/txt/pasteMeta.txt.tmpl @@ -6,10 +6,10 @@ with delete token: {{.RootURL}}/{{.Paste.Key}}{{.FileExt}}?deleteToken={{.Paste. with delete token: {{end -}}type: {{.Paste.Type}} state: {{.Paste.State}} -{{if .Paste.TimeCreated.IsZero -}} +{{if .Paste.CreatedAt.IsZero -}} created: unknown {{else -}} -created: {{.Paste.TimeCreated}} +created: {{.Paste.CreatedAt}} {{end -}} delete token: {{.CanDeleteString}} diff --git a/bindata.go b/bindata.go index 0540b52..6872e98 100644 --- a/bindata.go +++ b/bindata.go @@ -102,7 +102,7 @@ func cssMainCss() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "css/main.css", size: 512, mode: os.FileMode(420), modTime: time.Unix(1587564030, 0)} + info := bindataFileInfo{name: "css/main.css", size: 512, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -122,7 +122,7 @@ func jsCopyclipboardJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "js/copyclipboard.js", size: 1593, mode: os.FileMode(420), modTime: time.Unix(1587564030, 0)} + info := bindataFileInfo{name: "js/copyclipboard.js", size: 1593, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -142,7 +142,7 @@ func jsDragdropJs() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "js/dragdrop.js", size: 2066, mode: os.FileMode(420), modTime: time.Unix(1576493365, 0)} + info := bindataFileInfo{name: "js/dragdrop.js", size: 2066, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -162,7 +162,7 @@ func templatesHtmlBaseHtmlTmpl() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "templates/html/base.html.tmpl", size: 365, mode: os.FileMode(420), modTime: time.Unix(1576604408, 0)} + info := bindataFileInfo{name: "templates/html/base.html.tmpl", size: 365, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -202,12 +202,12 @@ func templatesHtmlIndexHtmlTmpl() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "templates/html/index.html.tmpl", size: 1786, mode: os.FileMode(420), modTime: time.Unix(1576794526, 0)} + info := bindataFileInfo{name: "templates/html/index.html.tmpl", size: 1786, mode: os.FileMode(420), modTime: time.Unix(1595849380, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _templatesHtmlPastemetaHtmlTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x94\xc1\x6e\xdb\x30\x0c\x86\xef\x7a\x0a\x42\x87\x36\x3d\x38\xbe\xb7\xb6\x07\x2c\xdd\x80\x61\x3d\x0c\x5d\x7b\xd9\x4d\xb1\x98\x5a\x8d\x22\x69\x12\xdd\xd6\x10\xf4\xee\x83\xec\x3a\xe9\xb6\x60\x4b\x2f\x01\x42\x51\x1f\xff\x9f\x22\x1d\xa3\xc4\x8d\x32\x08\x9c\x14\x69\xe4\x29\xb1\xf3\x18\x97\xdf\x44\x20\x5c\x7e\xc5\x21\xa5\x18\x97\x9f\x95\xc6\x4f\x2f\x94\xd2\x39\xec\x90\x84\x14\x24\xa0\x00\xdf\x87\x4e\x2b\xb3\x65\x31\xa2\x91\x29\x31\x76\x80\x75\x28\x64\x21\x9c\x43\x23\x33\xb2\x0a\xad\x57\x8e\x80\x06\x87\x35\x27\x7c\xa1\xf2\x51\x3c\x89\x29\xca\x21\xf8\xb6\xe6\xe5\x63\x28\x5b\xeb\x86\x56\x2b\xb7\xb6\xc2\xcb\xe5\x63\xe0\x20\x71\x83\xbe\xa9\xca\x29\xb5\x39\x56\x6b\x6d\xe5\x30\x16\x71\x1e\x41\xc9\x9a\x67\xca\x9d\x5d\xcd\x9c\x95\x35\x24\x94\x41\xcf\xa1\xd5\x22\x84\x9a\x77\x4a\x4a\x34\xbc\x61\xd5\xba\x27\xb2\x06\xac\x69\xb5\x6a\xb7\x7f\x5d\x5d\xe4\x5e\xdc\x5a\x4b\xf7\xb7\x37\x29\x95\xff\x68\xcc\x05\x6f\x56\xd6\x0d\x70\x7f\x7b\x03\x64\x61\x6f\xa2\x2a\xa7\x12\x0d\x54\xc1\x09\x73\x4c\xdf\x77\x12\xd4\x07\x9e\x4d\x3a\x61\x1a\x56\x95\xce\x63\x33\xda\x69\x58\x25\xa0\xf3\xb8\xa9\xf9\x89\x42\x78\x73\x62\x62\x55\x8a\x86\x15\x45\xc1\x62\x54\x1b\x10\x46\xc2\xc2\x20\xbc\x66\x67\x49\xf9\xd7\x2b\xf3\x00\x5c\xa2\x46\x42\xc9\x2f\x60\xb9\x12\xe6\x7a\xfc\xf7\xd1\x5a\x9d\x12\x7b\x56\xd4\xc1\x74\x0e\x64\xb7\x68\x2e\xe1\xbd\x8a\x3f\x4c\xd7\xef\xf2\xed\x7a\x9f\x76\x7d\x08\xbe\xc3\xd4\x09\xac\xd1\x77\x8c\xa8\x03\x42\x71\xdc\xc1\x99\xa6\xab\xde\x6c\x8d\x7d\x36\x67\x0f\x74\x35\xcd\xdc\x98\x9c\xe7\xf7\x12\xf6\xe0\xbb\xc1\x61\x4a\x2c\xe4\x6e\xbd\x09\x8f\xdd\x4b\x69\xea\xec\x9c\xaa\x76\xb8\xf2\x28\x08\xe5\xf2\x4b\xf8\x81\xde\x8e\xc0\x76\x0a\x5d\xc2\x6b\xbd\xb7\xca\xf6\x67\x87\x7a\x07\xc8\x88\x9f\x65\xfd\x2e\x3f\xc6\xc3\x33\x4d\x4f\x98\x97\x63\x63\xfd\x0e\x44\x4b\xca\x9a\xfa\xf5\x45\x4f\xe9\x7c\x5e\xf8\xce\xca\x9a\x3b\x1b\x28\x6f\x8c\x32\xae\x9f\x17\x39\xf4\xeb\x9d\x22\x0e\x46\xec\x70\x86\x72\x78\x12\xba\xc7\x9a\x4f\x1c\x70\x19\xca\x59\x8c\x05\xa8\x0d\x58\x0f\x0b\xfc\xf9\xbf\x31\x5b\x18\x4b\x7f\xcc\xda\xc5\x64\x54\x05\xb1\xd6\x28\x47\xdc\xec\x3e\x6f\x4c\x76\xb7\xdf\x9c\xf9\x1b\xf1\x2b\x00\x00\xff\xff\xf7\x78\x59\x94\xdc\x04\x00\x00") +var _templatesHtmlPastemetaHtmlTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x94\xc1\x6e\xdb\x38\x10\x86\xef\x7a\x8a\x01\x0f\x89\x73\x90\x75\x4f\x24\x2d\x76\x9d\x5d\x60\xd1\x1c\x8a\xd4\xe9\xa1\x37\x5a\x1c\xc7\x8c\x69\x92\x25\x47\x49\x0c\x82\xef\x5e\x50\xb4\x65\xa7\x71\x5b\xe7\x22\x40\xc3\xe1\xf7\xcf\x3f\x98\x61\x08\x02\x97\x52\x23\x30\x92\xa4\x90\xc5\x58\x5c\x86\x30\xfd\xcc\x3d\xe1\xf4\x13\x6e\x63\x0c\x61\xfa\x9f\x54\xf8\xef\x2b\xc5\x78\x09\x1b\x24\x2e\x38\x71\x28\xc1\xf5\x7e\xa5\xa4\x5e\x17\x21\xa0\x16\x31\x16\xc5\x01\xb6\x42\x2e\x4a\x6e\x2d\x6a\x91\x90\xb5\xef\x9c\xb4\x04\xb4\xb5\xd8\x30\xc2\x57\xaa\x9e\xf8\x33\xcf\x51\x06\xde\x75\x0d\xab\x9e\x7c\xd5\x19\xbb\xed\x94\xb4\x0b\xc3\x9d\x98\x3e\x79\x06\x02\x97\xe8\xda\xba\xca\xa9\xed\x29\xad\x85\x11\xdb\x41\xc4\x3a\x04\x29\x1a\x96\x28\x73\x33\xdb\x73\x66\x46\x13\x97\x1a\x1d\x83\x4e\x71\xef\x1b\xb6\x92\x42\xa0\x66\x6d\x51\x2f\x7a\x22\xa3\xc1\xe8\x4e\xc9\x6e\xfd\xee\xea\x24\xf5\xe2\xde\x18\x7a\xb8\xbf\x8b\xb1\xfa\x4d\x63\xae\x58\x3b\x33\x76\x0b\x0f\xf7\x77\x40\x06\x46\x13\x75\x95\x25\x5a\xa8\xbd\xe5\xfa\x54\x7d\x5f\x88\x53\xef\x59\x32\x69\xb9\x6e\x8b\xba\xb2\x0e\xdb\xc1\x4e\x5b\xd4\x1c\x56\x0e\x97\x0d\x3b\xb3\x10\xd6\x9e\x99\x58\x57\xbc\x2d\xca\xb2\x2c\x42\x90\x4b\xe0\x5a\xc0\x44\x23\xec\xb2\x53\x49\xe9\xeb\xa4\x7e\x04\x26\x50\x21\xa1\x60\x57\x30\x9d\x71\x7d\x3b\xfc\xfd\x63\x8c\x8a\xb1\x78\x91\xb4\x82\x7c\x0e\x64\xd6\xa8\xaf\xe1\xa3\x15\xff\x95\xaf\xcf\xd3\xed\x66\x4c\xbb\x3d\x04\x3f\x60\xea\x0c\xd6\xe0\x3b\x04\x54\x1e\xa1\x3c\xed\xe0\x42\xd1\x4d\xaf\xd7\xda\xbc\xe8\x8b\x47\xba\xc9\x33\x37\x24\xa7\xf9\xbd\x86\x11\x3c\xdf\x5a\x8c\xb1\xf0\xa9\x5b\x47\xe1\xa1\x7b\x31\xe6\xce\xee\x62\x33\x87\x9c\x50\xfc\x4d\xd3\xff\xfd\x37\x74\x66\xc0\x75\x39\x78\x0d\x3b\xb5\xe3\xba\xc6\xb3\x11\x3b\x22\x06\xf4\xbe\xa4\x63\x91\x07\x2b\xde\x8b\x28\xee\x09\xfa\x7c\x72\x52\xe9\x6d\xc2\x28\x37\xc2\x7e\x29\x97\xfb\x9a\xe4\xbe\x72\x25\xf3\xf9\x6e\x56\x80\xd3\x11\xea\x90\x38\x97\x1b\x7c\xc3\x7b\xdb\xf9\x10\x0e\x13\x96\xa7\x2f\xed\xf5\xd2\xb8\x0d\xf0\x8e\xa4\xd1\xcd\x6e\x18\xcf\x19\x9a\xf4\x56\xad\x8c\x68\x98\x35\x9e\xd2\xb2\x4b\x6d\xfb\xfd\x1b\xe4\xfb\xc5\x46\x12\x03\xcd\x37\xb8\x87\x32\x78\xe6\xaa\xc7\x86\x65\x0e\xd8\x04\x65\x45\x08\x25\xc8\x25\x18\x07\x13\xfc\xfe\xa7\x0d\x99\x68\x43\x3f\xad\xc9\x55\x36\x2a\x3d\x5f\x28\x14\x03\x6e\xef\x3e\x2d\x7b\x72\x37\x2e\xfd\xfe\x79\xfb\x11\x00\x00\xff\xff\x3a\xc6\xf5\xbb\x97\x05\x00\x00") func templatesHtmlPastemetaHtmlTmplBytes() ([]byte, error) { return bindataRead( @@ -222,7 +222,7 @@ func templatesHtmlPastemetaHtmlTmpl() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "templates/html/pasteMeta.html.tmpl", size: 1244, mode: os.FileMode(420), modTime: time.Unix(1589228935, 0)} + info := bindataFileInfo{name: "templates/html/pasteMeta.html.tmpl", size: 1431, mode: os.FileMode(420), modTime: time.Unix(1603641311, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -282,12 +282,12 @@ func templatesTxtIndexTxtTmpl() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "templates/txt/index.txt.tmpl", size: 684, mode: os.FileMode(420), modTime: time.Unix(1576794536, 0)} + info := bindataFileInfo{name: "templates/txt/index.txt.tmpl", size: 684, mode: os.FileMode(420), modTime: time.Unix(1590859748, 0)} a := &asset{bytes: bytes, info: info} return a, nil } -var _templatesTxtPastemetaTxtTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x51\xcd\x4a\x03\x31\x10\xbe\xe7\x29\x86\x78\x51\x30\xf1\x5e\xfc\x01\xdb\x15\xc4\x1e\xa4\xae\x17\x4f\x5d\x76\xa7\x36\x74\x4d\xea\x66\x96\x76\x09\xf3\xee\xb2\x49\xdd\x2d\xea\x41\xd0\x4b\x20\x33\xdf\xcc\xf7\x33\x21\xe8\x85\x73\xf4\xbc\x98\x33\x5f\x84\xa0\x1f\x0b\x4f\xa8\x1f\xb0\x63\x0e\x41\xdf\x99\x1a\xb3\x3d\x31\x0b\xa5\x94\x08\xc1\xac\xa0\xb0\x15\x9c\x5a\x84\x03\xf2\x89\x8a\xf8\x36\xc6\xbe\x82\xac\xb0\x46\xc2\x4a\x9e\x81\x9e\x16\x76\x16\x7f\xb7\xce\xd5\xcc\x62\x67\x68\x0d\xa9\x0f\xe4\x36\x68\x27\xf0\x4b\xee\x9b\x34\x95\xf7\x43\x57\x03\x6c\x36\x16\x99\x45\x08\x58\x7b\x04\xf5\x33\xd1\x65\x6b\x37\xd6\xed\xec\x75\x8f\xb3\x55\x0f\xa3\x6e\x8b\x51\x41\xda\x96\x77\x5b\x64\x16\xbe\x77\x73\x54\x8e\xee\xe2\x7a\xb3\xfa\x74\x9c\x9b\x37\x9c\x36\x58\x10\x56\xfa\xde\xbf\x60\xe3\x22\x6d\x99\x4a\x13\x38\x70\x1d\x4b\x1a\x7a\x23\xdf\xb8\x24\xa9\x4f\xaa\xc4\xb7\x80\x86\x18\x53\xc4\xcc\xe2\x6f\x67\x58\x2e\x97\xe2\x04\x72\x37\x24\xb4\x36\xfe\x6b\x0c\xe7\x80\x7b\x2c\x5b\xc2\x89\x28\xdb\xa6\x06\xa5\x1a\x7c\x6f\xd1\x13\xc8\x59\x36\xcf\xf2\x4c\x82\xfc\xbf\xdb\xc9\xa8\x29\x46\xc0\x2c\x3e\x02\x00\x00\xff\xff\x34\xf2\xf4\x10\x90\x02\x00\x00") +var _templatesTxtPastemetaTxtTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x51\x4f\x4b\x3b\x31\x10\xbd\xe7\x53\x0c\xf9\x5d\x7e\x82\x89\xf7\xe2\x1f\xb4\x5d\x41\xec\x41\xea\x7a\xf1\xd4\x65\x77\x6a\x43\x97\xa4\x6e\x66\x69\x97\x30\xdf\x5d\x36\xa9\xb1\xa8\x07\x41\x2f\x81\xcc\xbc\x79\xef\xcd\x9b\x10\xf4\xc2\x39\x7a\x5a\xcc\x99\xcf\x42\xd0\x0f\x95\x27\xd4\xf7\x38\x30\x87\xa0\x6f\x4d\x8b\xc5\x9e\x98\x85\x52\x4a\x84\x60\x56\x50\xd9\x06\xfe\x5b\x84\x03\xf2\x91\xaa\xf8\x76\xc6\xbe\x80\x6c\xb0\x45\xc2\x46\x9e\x80\x9e\x56\x76\x16\x7f\x37\xce\xb5\xcc\x62\x67\x68\x0d\xa9\x0f\xe4\x36\x68\x27\xf0\x43\xed\xab\x34\x55\x8e\x43\x17\x19\x36\xfb\x28\x32\x8b\x10\xb0\xf5\x08\xea\x7b\xa1\xf3\xde\x6e\xac\xdb\xd9\xcb\x11\x67\x9b\x11\x46\xc3\x16\xa3\x83\xc4\x56\x0e\x5b\x64\x16\x7e\xdc\xe6\xa8\x1c\xb7\x8b\xf4\x66\xf5\xbe\xf1\xb4\xc3\x8a\xb0\xb9\x26\x7d\xe7\x9f\xb1\x73\x51\xb4\x4e\xc5\x09\x1c\x94\x8e\x0d\xe5\x5e\xa6\xcd\x14\xc9\x79\x72\x24\xbe\x84\x93\x23\x4c\xf1\x32\x8b\xdf\x9d\x60\xb9\x5c\x8a\x7f\x50\xba\x9c\xce\xda\xf8\xcf\x11\x9c\x02\xee\xb1\xee\x09\x27\xa2\xee\xbb\x16\x94\xea\xf0\xb5\x47\x4f\x20\x67\xc5\xbc\x28\x0b\x09\xf2\xef\xee\x26\xa3\xa7\x18\x01\xb3\x78\x0b\x00\x00\xff\xff\x81\xcd\x43\x9e\x8c\x02\x00\x00") func templatesTxtPastemetaTxtTmplBytes() ([]byte, error) { return bindataRead( @@ -302,7 +302,7 @@ func templatesTxtPastemetaTxtTmpl() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "templates/txt/pasteMeta.txt.tmpl", size: 656, mode: os.FileMode(420), modTime: time.Unix(1589229127, 0)} + info := bindataFileInfo{name: "templates/txt/pasteMeta.txt.tmpl", size: 652, mode: os.FileMode(420), modTime: time.Unix(1603638601, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/cmd/rushlink-migrate-db/main.go b/cmd/rushlink-migrate-db/main.go new file mode 100644 index 0000000..35f0f53 --- /dev/null +++ b/cmd/rushlink-migrate-db/main.go @@ -0,0 +1,110 @@ +package main + +import ( + "flag" + "log" + "time" + + "gitea.hashru.nl/dsprenkels/rushlink/internal/boltdb" + "gitea.hashru.nl/dsprenkels/rushlink/internal/db" + bolt "go.etcd.io/bbolt" + "gorm.io/gorm" +) + +var ( + boltDBPath = flag.String("boltdb", "", "location of the bolt database file") + fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored") +) + +func main() { + flag.Parse() + + boltFileStore, err := boltdb.OpenFileStore(*fileStorePath) + if err != nil { + log.Fatalln(err) + } + boltDB, err := boltdb.OpenDB(*boltDBPath, boltFileStore) + if err != nil { + log.Fatalln(err) + } + defer boltDB.Close() + sqlDB, err := db.OpenDBFromEnvironment() + if err != nil { + log.Fatalln(err) + } + + // Migrate database schema + m := db.Gormigrate(sqlDB) + if err := m.MigrateTo("202010251337"); err != nil { + log.Fatalln(err) + } + + // Migrate all files in filestorage + if err := sqlDB.Transaction(func(sqlTx *gorm.DB) error { + return boltDB.Bolt.View(func(boltTx *bolt.Tx) error { + // Migrate all the file uploads + allFUs, err := boltdb.AllFileUploads(boltTx) + if err != nil { + return err + } + var fusDeleted uint + var sqlFUs []db.FileUpload + for _, fu := range allFUs { + isDeleted := fu.State == boltdb.FileUploadStateDeleted + sqlFU := db.FileUpload{ + State: db.FileUploadState(int(fu.State)), + PubID: fu.ID, + FileName: fu.FileName, + ContentType: fu.ContentType, + Checksum: fu.Checksum, + DeletedAt: deletedAt(isDeleted), + } + sqlFUs = append(sqlFUs, sqlFU) + if isDeleted { + fusDeleted++ + } + } + log.Printf("migrating %v file uploads (of which %v deleted)", len(sqlFUs), fusDeleted) + sqlTx.Create(sqlFUs) + + // Migrate all the pastes. + allPastes, err := boltdb.AllPastes(boltTx) + if err != nil { + return err + } + var pastesDeleted uint + var sqlPastes []db.Paste + for _, paste := range allPastes { + isDeleted := paste.State == boltdb.PasteStateDeleted + sqlPaste := db.Paste{ + Type: db.PasteType(int(paste.Type)), + State: db.PasteState(int(paste.State)), + Content: paste.Content, + Key: paste.Key, + DeleteToken: paste.DeleteToken, + CreatedAt: paste.TimeCreated, + DeletedAt: deletedAt(isDeleted), + } + sqlPastes = append(sqlPastes, sqlPaste) + if isDeleted { + pastesDeleted++ + } + } + log.Printf("migrating %v pastes (of which %v deleted)", len(sqlPastes), pastesDeleted) + sqlTx.Create(sqlPastes) + + return nil + }) + }); err != nil { + log.Fatalln(err) + } + + log.Println("migration successful! :D") +} + +func deletedAt(isDeleted bool) gorm.DeletedAt { + if isDeleted { + return gorm.DeletedAt{Time: time.Now(), Valid: true} + } + return gorm.DeletedAt{} +} diff --git a/cmd/rushlink/main.go b/cmd/rushlink/main.go index 36ae1cc..d21b000 100644 --- a/cmd/rushlink/main.go +++ b/cmd/rushlink/main.go @@ -6,11 +6,10 @@ import ( "gitea.hashru.nl/dsprenkels/rushlink" "gitea.hashru.nl/dsprenkels/rushlink/internal/db" + "github.com/pkg/errors" ) var ( - databasePath = flag.String("database", "", "location of the database file") - fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored") httpListen = flag.String("listen", "127.0.0.1:8000", "listen address (host:port)") metricsListen = flag.String("metrics_listen", "127.0.0.1:58614", "listen address for metrics (host:port)") rootURL = flag.String("root_url", "", "host root (example: 'https://example.com', uses an educated guess if omitted)") @@ -19,15 +18,19 @@ var ( func main() { flag.Parse() - filestore, err := db.OpenFileStore(*fileStorePath) + filestore, err := db.OpenFileStoreFromEnvironment() if err != nil { log.Fatalln(err) } - database, err := db.OpenDB(*databasePath, filestore) + database, err := db.OpenDBFromEnvironment() if err != nil { log.Fatalln(err) } - defer database.Close() + + migrate := db.Gormigrate(database) + if err := migrate.Migrate(); err != nil { + log.Fatal(errors.Wrap(err, "migrating database")) + } go rushlink.StartMetricsServer(*metricsListen, database, filestore) rushlink.StartMainServer(*httpListen, database, filestore, *rootURL) diff --git a/go.mod b/go.mod index 65f1f6f..1041fe4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect + github.com/go-gormigrate/gormigrate/v2 v2.0.0 github.com/golang/protobuf v1.4.2 // indirect github.com/google/go-cmp v0.4.1 // indirect github.com/google/uuid v1.1.1 @@ -14,4 +15,7 @@ require ( go.etcd.io/bbolt v1.3.4 golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect google.golang.org/protobuf v1.24.0 // indirect + gorm.io/driver/postgres v1.0.0 + gorm.io/driver/sqlite v1.1.3 + gorm.io/gorm v1.20.5 ) diff --git a/go.sum b/go.sum index a94ed1a..7ae0316 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -12,18 +14,30 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/go-bindata/go-bindata v1.0.0 h1:DZ34txDXWn1DyWa+vQf7V9ANc2ILTtrEjtlsdJRF26M= +github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= +github.com/go-gormigrate/gormigrate v1.6.0 h1:JvKpFsbVPLfIPPkIrZq1zq19PLFT4DVSnL5BiyE1P0E= +github.com/go-gormigrate/gormigrate/v2 v2.0.0 h1:e2A3Uznk4viUC4UuemuVgsNnvYZyOA8B3awlYk3UioU= +github.com/go-gormigrate/gormigrate/v2 v2.0.0/go.mod h1:YuVJ+D/dNt4HWrThTBnjgZuRbt7AuwINeg4q52ZE3Jw= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -47,21 +61,92 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8= +github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po= +github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= +github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY= +github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA= +github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= +github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= +github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw= +github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= +github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA= +github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -77,10 +162,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= -github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -90,43 +171,70 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -137,22 +245,39 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -176,10 +301,26 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw= +gorm.io/driver/postgres v1.0.0 h1:Yh4jyFQ0a7F+JPU0Gtiam/eKmpT/XFc1FKxotGqc6FM= +gorm.io/driver/postgres v1.0.0/go.mod h1:wtMFcOzmuA5QigNsgEIb7O5lhvH1tHAF1RbWmLWV4to= +gorm.io/driver/sqlite v1.1.1/go.mod h1:hm2olEcl8Tmsc6eZyxYSeznnsDaMqamBvEXLNtBg4cI= +gorm.io/driver/sqlite v1.1.3 h1:BYfdVuZB5He/u9dt4qDpZqiqDJ6KhPqs5QUqsr/Eeuc= +gorm.io/driver/sqlite v1.1.3/go.mod h1:AKDgRWk8lcSQSw+9kxCJnX/yySj8G3rdwYlU57cB45c= +gorm.io/driver/sqlserver v1.0.2/go.mod h1:gb0Y9QePGgqjzrVyTQUZeh9zkd5v0iz71cM1B4ZycEY= +gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.0/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.20.5 h1:g3tpSF9kggASzReK+Z3dYei1IJODLqNUbOjSuCczY8g= +gorm.io/gorm v1.20.5/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/handlers.go b/handlers.go index a058838..3dd58aa 100644 --- a/handlers.go +++ b/handlers.go @@ -15,7 +15,6 @@ import ( "github.com/google/uuid" "github.com/gorilla/mux" "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" ) const ( @@ -98,7 +97,8 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request key := vars["key"] var p *db.Paste var fu *db.FileUpload - err := rl.db.Bolt.View(func(tx *bolt.Tx) error { + var notFound bool + err := rl.db.Transaction(func(tx *db.Database) error { var err error p, err = db.GetPaste(tx, key) if err != nil { @@ -114,6 +114,9 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request } return nil }) + if notFound { + err = db.ErrPasteDoesNotExist + } if err != nil { status := db.ErrHTTPStatusCode(err) if status == http.StatusInternalServerError { @@ -122,7 +125,6 @@ func (rl *rushlink) viewPasteHandlerFlags(w http.ResponseWriter, r *http.Request rl.renderError(w, r, status, err.Error()) return } - rl.viewPasteHandlerInner(w, r, flags, p, fu) } @@ -282,7 +284,7 @@ func (rl *rushlink) newPasteHandler(w http.ResponseWriter, r *http.Request) { func (rl *rushlink) newFileUploadPasteHandler(w http.ResponseWriter, r *http.Request, file multipart.File, header multipart.FileHeader) { var fu *db.FileUpload var paste *db.Paste - if err := rl.db.Bolt.Update(func(tx *bolt.Tx) error { + if err := rl.db.Transaction(func(tx *db.Database) error { var err error fu, err = db.NewFileUpload(rl.fs, file, header.Filename) if err != nil { @@ -292,7 +294,7 @@ func (rl *rushlink) newFileUploadPasteHandler(w http.ResponseWriter, r *http.Req panic(errors.Wrap(err, "saving fileUpload in db")) } - paste, err = shortenFileUploadID(tx, fu.ID) + paste, err = shortenFileUploadID(tx, fu.PubID) return err }); err != nil { panic(err) @@ -317,7 +319,7 @@ func (rl *rushlink) newRedirectPasteHandler(w http.ResponseWriter, r *http.Reque } var paste *db.Paste - if err := rl.db.Bolt.Update(func(tx *bolt.Tx) error { + if err := rl.db.Transaction(func(tx *db.Database) error { var err error paste, err = shortenURL(tx, userURL) return err @@ -340,7 +342,7 @@ func (rl *rushlink) deletePasteHandler(w http.ResponseWriter, r *http.Request) { var errorCode int var paste *db.Paste - if err := rl.db.Bolt.Update(func(tx *bolt.Tx) error { + if err := rl.db.Transaction(func(tx *db.Database) error { var err error paste, err = db.GetPaste(tx, key) if err != nil { @@ -372,19 +374,19 @@ func (rl *rushlink) deletePasteHandler(w http.ResponseWriter, r *http.Request) { // // Returns the new paste key if the fileUpload was successfully added to the // database -func shortenFileUploadID(tx *bolt.Tx, id uuid.UUID) (*db.Paste, error) { +func shortenFileUploadID(tx *db.Database, id uuid.UUID) (*db.Paste, error) { return shorten(tx, db.PasteTypeFileUpload, id[:]) } // Add a new URL to the database // // Returns the new paste key if the url was successfully shortened -func shortenURL(tx *bolt.Tx, userURL *url.URL) (*db.Paste, error) { +func shortenURL(tx *db.Database, userURL *url.URL) (*db.Paste, error) { return shorten(tx, db.PasteTypeRedirect, []byte(userURL.String())) } // Add a paste (of any kind) to the database with arbitrary content. -func shorten(tx *bolt.Tx, ty db.PasteType, content []byte) (*db.Paste, error) { +func shorten(tx *db.Database, ty db.PasteType, content []byte) (*db.Paste, error) { // Generate the paste key var keyEntropy int if ty == db.PasteTypeFileUpload || ty == db.PasteTypePaste { @@ -408,7 +410,6 @@ func shorten(tx *bolt.Tx, ty db.PasteType, content []byte) (*db.Paste, error) { Content: content, Key: pasteKey, DeleteToken: deleteToken, - TimeCreated: time.Now().UTC(), } if err := p.Save(tx); err != nil { return nil, err diff --git a/handlers_test.go b/handlers_test.go index de5ff53..5cfc6a1 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -15,7 +15,6 @@ import ( "gitea.hashru.nl/dsprenkels/rushlink/internal/db" "github.com/gorilla/mux" - "go.etcd.io/bbolt" ) // createTemporaryRouter initializes a rushlink instance, with temporary @@ -35,16 +34,18 @@ func createTemporaryRouter(t *testing.T) (*mux.Router, *rushlink) { if err != nil { t.Fatalf("opening temporary filestore: %s\n", err) } - databasePath := filepath.Join(tempDir, "rushlink.db") - database, err := db.OpenDB(databasePath, fileStore) + + os.Setenv(db.EnvDatabaseDriver, "sqlite") + os.Setenv(db.EnvDatabasePath, "file::memory:?cache=shared") + + database, err := db.OpenDBFromEnvironment() if err != nil { - t.Fatalf("opening temporary database: %s\n", err) + t.Fatalf("opening temporary database: %v\n", err) + } + database.Debug() + if err := db.Gormigrate(database).Migrate(); err != nil { + t.Fatalf("migration failed: %v\n", err) } - t.Cleanup(func() { - if err := database.Close(); err != nil { - t.Errorf("closing database: %d\n", err) - } - }) // *.invalid. is guaranteed not to exist (RFC 6761). rootURL, err := url.Parse("https://rushlink.invalid") @@ -135,7 +136,7 @@ func TestIssue53(t *testing.T) { checkStatusCode(t, rr, http.StatusFound) // Check that any attempt to do directory traversal has failed. - rl.db.Bolt.View(func(tx *bbolt.Tx) error { + rl.db.Transaction(func(tx *db.Database) error { fus, err := db.AllFileUploads(tx) if err != nil { t.Fatal(err) diff --git a/internal/boltdb/db.go b/internal/boltdb/db.go new file mode 100644 index 0000000..f433d81 --- /dev/null +++ b/internal/boltdb/db.go @@ -0,0 +1,204 @@ +package boltdb + +import ( + "bytes" + "fmt" + "io" + "log" + "net/http" + "os" + "time" + + "github.com/pkg/errors" + + bolt "go.etcd.io/bbolt" + + gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" +) + +// Database is the main rushlink database type. +// +// Open a database using DB.Open() and close it in the end using DB.Close(). +// Only one instance of DB should exist in a program at any moment. +type Database struct { + Bolt *bolt.DB +} + +// CurrentMigrateVersion holds the current "migrate version". +// +// If we alter the database format, we bump this number and write a new +// database migration in migrate(). +const CurrentMigrateVersion = 3 + +// BucketConf holds the name for the "configuration" bucket. +// +// This bucket holds the database version, secret site-wide keys, etc. +const BucketConf = "conf" + +// BucketPastes holds the name for the pastes bucket. +const BucketPastes = "pastes" + +// BucketFileUpload holds the name for the file-upload bucket. +const BucketFileUpload = "fileUpload" + +// KeyMigrateVersion stores the current migration version. If this value is less than +// CurrentMigrateVersion, the database has to be migrated. +const KeyMigrateVersion = "migrate_version" + +// OpenDB opens a database file located at path. +func OpenDB(path string, fs *FileStore) (*Database, error) { + if path == "" { + return nil, errors.New("database not set") + } + + db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return nil, errors.Wrapf(err, "failed to open database at '%v'", path) + } + if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil { + return nil, err + } + return &Database{db}, nil +} + +// Close the bolt database +func (db *Database) Close() error { + if db == nil { + panic("no open database") + } + return db.Bolt.Close() +} + +// Initialize and migrate the database to the current version +func migrate(tx *bolt.Tx, fs *FileStore) error { + // Guidelines for error handling: + // - Errors based on malformed *structure* should be fatal! + // - Errors based on malformed *data* should print a warning + // (and if possible try to fix the error). + + dbVersion, err := dbVersion(tx) + if err != nil { + return err + } + + // Migrate the database to version 1 + if dbVersion < 1 { + log.Println("migrating database to version 1") + // Create conf bucket + _, err := tx.CreateBucket([]byte(BucketConf)) + if err != nil { + return err + } + // Create paste bucket + _, err = tx.CreateBucket([]byte(BucketPastes)) + if err != nil { + return err + } + // Update the version number + if err := setDBVersion(tx, 1); err != nil { + return err + } + } + + if dbVersion < 2 { + log.Println("migrating database to version 2") + // Create fileUpload bucket + _, err := tx.CreateBucket([]byte(BucketFileUpload)) + if err != nil { + return err + } + // Update the version number + if err := setDBVersion(tx, 2); err != nil { + return err + } + } + + if dbVersion < 3 { + log.Println("migrating database to version 3") + // In this version, we changed te way how Content-Types are being + // stored. Previously, we allowed clients to provide their own + // Content-Types for files, using the Content-Disposition header in + // multipart forms. The new way detects these types using + // http.DetectContentType. + // + // Scan through all the FileUploads and update their ContentTypes. + bucket := tx.Bucket([]byte(BucketFileUpload)) + cursor := bucket.Cursor() + var id, storedBytes []byte + id, storedBytes = cursor.First() + for id != nil { + fu, err := decodeFileUpload(storedBytes) + if err != nil { + log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id)) + id, storedBytes = cursor.Next() + continue + } + if fu.State != FileUploadStatePresent { + id, storedBytes = cursor.Next() + continue + } + filePath := fu.Path(fs) + file, err := os.Open(fu.Path(fs)) + if err != nil { + log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath)) + id, storedBytes = cursor.Next() + continue + } + var buf bytes.Buffer + buf.Grow(512) + io.CopyN(&buf, file, 512) + contentType := http.DetectContentType(buf.Bytes()) + if contentType != fu.ContentType { + fu.ContentType = contentType + fu.Save(tx) + cursor.Seek(id) + } + id, storedBytes = cursor.Next() + } + // Update the version number + if err := setDBVersion(tx, 3); err != nil { + return err + } + } + + return nil +} + +// Get the current migrate version from the database +func dbVersion(tx *bolt.Tx) (int, error) { + conf := tx.Bucket([]byte(BucketConf)) + if conf == nil { + return 0, nil + } + dbVersionBytes := conf.Get([]byte(KeyMigrateVersion)) + if dbVersionBytes == nil { + return 0, nil + } + + // Version was already stored + var dbVersion int + if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil { + return 0, err + } + if dbVersion == 0 { + return 0, fmt.Errorf("database version is invalid (%v)", dbVersion) + } + if dbVersion > CurrentMigrateVersion { + return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion) + } + return dbVersion, nil +} + +// Update the current migrate version in the database +func setDBVersion(tx *bolt.Tx, version int) error { + conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf)) + if err != nil { + return err + } + + versionBytes, err := gobmarsh.Marshal(version) + if err != nil { + return err + } + return conf.Put([]byte(KeyMigrateVersion), versionBytes) +} diff --git a/internal/boltdb/fileupload.go b/internal/boltdb/fileupload.go new file mode 100644 index 0000000..adb2dbf --- /dev/null +++ b/internal/boltdb/fileupload.go @@ -0,0 +1,256 @@ +package boltdb + +import ( + "bytes" + "encoding/hex" + "hash/crc32" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + + "github.com/google/uuid" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" + + gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" +) + +// Use the Castagnoli checksum because of the acceleration on Intel CPUs +var checksumTable = crc32.MakeTable(crc32.Castagnoli) + +// FileStore holds the path to a file storage location. +type FileStore struct { + path string +} + +// FileUploadState determines the current state of a FileUpload object. +type FileUploadState int + +// FileUpload models an uploaded file. +type FileUpload struct { + // State of the FileUpload (present/deleted/etc). + State FileUploadState + + // ID identifies this FileUpload. + ID uuid.UUID + + // FileName contains the original filename of this FileUpload. + FileName string + + // Content type as determined by http.DetectContentType. + ContentType string + + // Checksum holds a crc32c checksum of the file. + // + // This checksum is only meant to allow for the detection of random + // database corruption. + Checksum uint32 +} + +const ( + dirMode os.FileMode = 0750 + fileMode os.FileMode = 0640 +) + +const ( + // FileUploadStateUndef is an undefined FileUpload. + FileUploadStateUndef FileUploadState = 0 + // FileUploadStatePresent denotes the normal (existing) state. + FileUploadStatePresent FileUploadState = 1 + // FileUploadStateDeleted denotes a deleted state. + FileUploadStateDeleted FileUploadState = 2 +) + +func (t FileUploadState) String() string { + switch t { + case FileUploadStateUndef: + return "unknown" + case FileUploadStatePresent: + return "present" + case FileUploadStateDeleted: + return "deleted" + default: + return "invalid" + } +} + +// OpenFileStore opens the file storage at path. +func OpenFileStore(path string) (*FileStore, error) { + if path == "" { + return nil, errors.New("file-store not set") + } + + // Try to create the file store directory if it does not yet exist + if err := os.MkdirAll(path, dirMode); err != nil { + return nil, errors.Wrap(err, "creating file store directory") + } + + return &FileStore{path[:]}, nil +} + +// Path returns the path of the FileStore root. +func (fs *FileStore) Path() string { + return fs.path +} + +// filePath resolves the path of a file in the FileStore given some id and filename. +func (fs *FileStore) filePath(id uuid.UUID, fileName string) string { + if fs.path == "" { + panic("fileStoreDir called while the file store path has not been set") + } + return path.Join(fs.path, hex.EncodeToString(id[:]), fileName) +} + +// NewFileUpload creates a new FileUpload object. +// +// Internally, this function detects the type of the file stored in `r` using +// `http.DetectContentType`. +func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) { + // Generate a file ID + id, err := uuid.NewRandom() + if err != nil { + return nil, errors.Wrap(err, "generating UUID") + } + + // Construct a checksum for this file + hash := crc32.New(checksumTable) + tee := io.TeeReader(r, hash) + + // Detect the file type + var tmpBuf bytes.Buffer + tmpBuf.Grow(512) + io.CopyN(&tmpBuf, tee, 512) + contentType := http.DetectContentType(tmpBuf.Bytes()) + + // Open the file on disk for writing + baseName := filepath.Base(fileName) + filePath := fs.filePath(id, baseName) + if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil { + return nil, errors.Wrap(err, "creating file dir") + } + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode) + if err != nil { + return nil, errors.Wrap(err, "opening file") + } + defer file.Close() + + // Write the file to disk + _, err = io.Copy(file, &tmpBuf) + if err != nil { + return nil, errors.Wrap(err, "writing to file") + } + _, err = io.Copy(file, tee) + if err != nil { + return nil, errors.Wrap(err, "writing to file") + } + + fu := &FileUpload{ + State: FileUploadStatePresent, + ID: id, + FileName: baseName, + ContentType: contentType, + Checksum: hash.Sum32(), + } + return fu, nil +} + +// GetFileUpload tries to retrieve a FileUpload object from the bolt database. +func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) { + bucket := tx.Bucket([]byte(BucketFileUpload)) + if bucket == nil { + return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload) + } + storedBytes := bucket.Get(id[:]) + if storedBytes == nil { + return nil, nil + } + return decodeFileUpload(storedBytes) +} + +// AllFileUploads tries to retrieve all FileUpload objects from the bolt database. +func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) { + bucket := tx.Bucket([]byte(BucketFileUpload)) + if bucket == nil { + return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload) + } + var fus []FileUpload + err := bucket.ForEach(func(_, storedBytes []byte) error { + fu, err := decodeFileUpload(storedBytes) + if err != nil { + return err + } + fus = append(fus, *fu) + return nil + }) + if err != nil { + return nil, err + } + return fus, nil +} + +func decodeFileUpload(storedBytes []byte) (*FileUpload, error) { + fu := &FileUpload{} + err := gobmarsh.Unmarshal(storedBytes, fu) + return fu, err +} + +// Save saves a FileUpload in the database. +func (fu *FileUpload) Save(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketFileUpload)) + if bucket == nil { + return errors.Errorf("bucket %v does not exist", BucketFileUpload) + } + + buf, err := gobmarsh.Marshal(fu) + if err != nil { + return errors.Wrap(err, "encoding for database failed") + } + if err := bucket.Put(fu.ID[:], buf); err != nil { + return errors.Wrap(err, "database transaction failed") + } + return nil +} + +// Delete deletes a FileUpload from the database. +func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error { + // Remove the file in the backend + filePath := fu.Path(fs) + if err := os.Remove(filePath); err != nil { + return err + } + + // Update the file in the server + if err := (&FileUpload{ + ID: fu.ID, + State: FileUploadStateDeleted, + }).Save(tx); err != nil { + return err + } + + // Cleanup the parent directory + wrap := "deletion succeeded, but removing the file directory has failed" + return errors.Wrap(os.Remove(path.Dir(filePath)), wrap) +} + +// Path returns the path to this FileUpload in the FileStore provided in fs. +func (fu *FileUpload) Path(fs *FileStore) string { + return fs.filePath(fu.ID, fu.FileName) +} + +// URL returns the URL for the FileUpload. +func (fu *FileUpload) URL() *url.URL { + rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName + urlParse, err := url.Parse(rawurl) + if err != nil { + panic("could not construct /uploads/ url") + } + return urlParse +} + +// Ext returns the extension of the file attached to this FileUpload. +func (fu *FileUpload) Ext() string { + return filepath.Ext(fu.FileName) +} diff --git a/internal/boltdb/paste.go b/internal/boltdb/paste.go new file mode 100644 index 0000000..50f2194 --- /dev/null +++ b/internal/boltdb/paste.go @@ -0,0 +1,345 @@ +package boltdb + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "net/http" + "net/url" + "strings" + "time" + + gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" + "github.com/google/uuid" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" +) + +// PasteType describes the type of Paste (i.e. file, redirect, [...]). +type PasteType int + +// PasteState describes the state of a Paste (i.e. present, deleted, [...]). +type PasteState int + +// Paste describes the main Paste model in the database. +type Paste struct { + Type PasteType + State PasteState + Content []byte + Key string + DeleteToken string + TimeCreated time.Time +} + +// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen +var ReservedPasteKeys = []string{"xd42", "example"} + +// Note: we use iota here. That means removals of PasteType* are not allowed, +// because this changes the value of the constant. Please add the comment +// "// deprecated" if you want to remove the constant. Additions are only +// allowed at the bottom of this block, for the same reason. +const ( + PasteTypeUndef PasteType = iota + // PasteTypePaste is as of yet unused. It is still unclear if this type + // will ever get a proper meaning. + PasteTypePaste + PasteTypeRedirect + PasteTypeFileUpload +) + +// Note: we use iota here. See the comment above PasteType* +const ( + PasteStateUndef PasteState = iota + PasteStatePresent + PasteStateDeleted +) + +// minKeyLen specifies the mimimum length of a paste key. +const minKeyLen = 4 + +var ( + // ErrKeyInvalidChar occurs when a key contains an invalid character. + ErrKeyInvalidChar = errors.New("invalid character in key") + // ErrKeyInvalidLength occurs when a key embeds a length that is incorrect. + ErrKeyInvalidLength = errors.New("key length encoding is incorrect") + // ErrPasteDoesNotExist occurs when a key does not exist in the database. + ErrPasteDoesNotExist = errors.New("url key not found in the database") +) + +// ErrHTTPStatusCode returns the HTTP status code that should correspond to +// the provided error. +// server error, or false if it is not. +func ErrHTTPStatusCode(err error) int { + switch err { + case nil: + return 0 + case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist: + return http.StatusNotFound + } + return http.StatusInternalServerError +} + +// Base64 encoding and decoding +var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" +var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding) + +func (t PasteType) String() string { + switch t { + case PasteTypeUndef: + return "unknown" + case PasteTypePaste: + return "paste" + case PasteTypeRedirect: + return "redirect" + case PasteTypeFileUpload: + return "file" + default: + return "invalid" + } +} + +func (t PasteState) String() string { + switch t { + case PasteStateUndef: + return "unknown" + case PasteStatePresent: + return "present" + case PasteStateDeleted: + return "deleted" + default: + return "invalid" + } +} + +// GetPaste retrieves a paste from the database. +func GetPaste(tx *bolt.Tx, key string) (*Paste, error) { + if err := ValidatePasteKey(key); err != nil { + return nil, err + } + return GetPasteNoValidate(tx, key) +} + +// ValidatePasteKey validates the format of the key that has +func ValidatePasteKey(key string) error { + internalLen := minKeyLen + countingOnes := true + for _, ch := range key { + limb := strings.IndexRune(base64Alphabet, ch) + if limb == -1 { + return ErrKeyInvalidChar + } + for i := 5; i >= 0 && countingOnes; i-- { + if (limb>>uint(i))&0x1 == 0 { + countingOnes = false + break + } + internalLen++ + } + } + if internalLen != len(key) { + return ErrKeyInvalidLength + } + return nil +} + +// GetPasteNoValidate retrieves a paste from the database without validating +// the key format first. +func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) { + pastesBucket := tx.Bucket([]byte(BucketPastes)) + if pastesBucket == nil { + return nil, errors.Errorf("bucket %v does not exist", BucketPastes) + } + storedBytes := pastesBucket.Get([]byte(key)) + if storedBytes == nil { + return nil, ErrPasteDoesNotExist + } + return decodePaste(storedBytes) +} + +func decodePaste(storedBytes []byte) (*Paste, error) { + p := &Paste{} + err := gobmarsh.Unmarshal(storedBytes, p) + return p, err +} + +// Save saves this Paste to the database. +func (p *Paste) Save(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketPastes)) + if bucket == nil { + return errors.Errorf("bucket %v does not exist", BucketPastes) + } + + buf, err := gobmarsh.Marshal(p) + if err != nil { + return errors.Wrap(err, "encoding for database failed") + } + if err := bucket.Put([]byte(p.Key), buf); err != nil { + return errors.Wrap(err, "database transaction failed") + } + return nil +} + +// Delete deletes this Paste from the database. +func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error { + // Remove the (maybe) attached file + if p.Type == PasteTypeFileUpload { + fuID, err := uuid.FromBytes(p.Content) + if err != nil { + return errors.Wrap(err, "failed to parse uuid") + } + fu, err := GetFileUpload(tx, fuID) + if err != nil { + return errors.Wrap(err, "failed to find file in database") + } + if err := fu.Delete(tx, fs); err != nil { + return errors.Wrap(err, "failed to remove file") + } + } + + // Replace the old paste with a new empty paste + p.Type = PasteTypeUndef + p.State = PasteStateDeleted + p.Content = []byte{} + if err := p.Save(tx); err != nil { + return errors.Wrap(err, "failed to delete paste in database") + } + return nil +} + +// RedirectURL returns the URL from this paste. +// +// This function assumes that the paste is valid. If the paste struct is +// corrupted in some way, this function will panic. +func (p *Paste) RedirectURL() *url.URL { + if p.Type != PasteTypeRedirect { + panic("expected p.Type to be PasteTypeRedirect") + } + rawurl := string(p.Content) + urlParse, err := url.Parse(rawurl) + if err != nil { + panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key)) + } + return urlParse +} + +// GeneratePasteKey generates a new paste key. It will ensure that the newly +// generated paste key does not already exist in the database. +// The running time of this function is in O(log N), where N is the amount of +// keys stored in the url-shorten database. +// In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum +// guessing entropy of the generated key. +func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) { + pastesBucket := tx.Bucket([]byte(BucketPastes)) + if pastesBucket == nil { + return "", errors.Errorf("bucket %v does not exist", BucketPastes) + } + + epoch := 0 + var key string + for { + var err error + key, err = generatePasteKeyInner(epoch, minimumEntropy) + if err != nil { + return "", errors.Wrap(err, "url-key generation failed") + } + + found := pastesBucket.Get([]byte(key)) + if found == nil { + break + } + + isReserved := false + for _, reservedKey := range ReservedPasteKeys { + if strings.HasPrefix(key, reservedKey) { + isReserved = true + break + } + } + if !isReserved { + break + } + + epoch++ + } + return key, nil +} + +// generatePasteKeyInner generates a new paste key, but leaves the +// uniqueness and is-reserved checks to the caller. That is, it only +// generates a random key in the correct (syntactical) format. +// Both epoch and entropy can be used to set the key length. Epoch is used +// to prevent collisions in retrying to generate new keys. Entropy (in bits) +// is used to ensure that a new key has at least some amount of guessing +// entropy. +func generatePasteKeyInner(epoch, entropy int) (string, error) { + entropyEpoch := entropy + entropyEpoch -= minKeyLen * 6 // First 4 characters provide 24 bits. + entropyEpoch++ // One bit less because of '0' bit. + entropyEpoch = (entropyEpoch-1)/5 + 1 // 5 bits for every added epoch. + if epoch < entropyEpoch { + epoch = entropyEpoch + } + urlKey := make([]byte, minKeyLen+epoch) + _, err := rand.Read(urlKey) + if err != nil { + return "", err + } + // Put all the values in the range 0..64 for easier base64-encoding + for i := 0; i < len(urlKey); i++ { + urlKey[i] &= 0x3F + } + // Implement truncate-resistance by forcing the prefix to + // 0b111110xxxxxxxxxx + // ^----- {epoch} ones followed by a single 0 + // + // Example when epoch is 1: prefix is 0b10. + i := 0 + for i < epoch { + // Set this bit to 1 + limb := i / 6 + bit := i % 6 + urlKey[limb] |= 1 << uint(5-bit) + i++ + } + // Finally set the next bit to 0 + limb := i / 6 + bit := i % 6 + urlKey[limb] &= ^(1 << uint(5-bit)) + + // Convert this ID to a canonical base64 notation + for i := range urlKey { + urlKey[i] = base64Alphabet[urlKey[i]] + } + return string(urlKey), nil +} + +// GenerateDeleteToken generates a new (random) delete token. +func GenerateDeleteToken() (string, error) { + var deleteToken [16]byte + _, err := rand.Read(deleteToken[:]) + if err != nil { + return "", err + } + return hex.EncodeToString(deleteToken[:]), nil +} + +// AllPastes tries to retrieve all the Paste objects from the database. +func AllPastes(tx *bolt.Tx) ([]Paste, error) { + bucket := tx.Bucket([]byte(BucketPastes)) + if bucket == nil { + return nil, errors.Errorf("bucket %v does not exist", BucketPastes) + } + var ps []Paste + err := bucket.ForEach(func(_, storedBytes []byte) error { + p, err := decodePaste(storedBytes) + if err != nil { + return err + } + ps = append(ps, *p) + return nil + }) + if err != nil { + return nil, err + } + return ps, nil +} diff --git a/internal/boltdb/paste_test.go b/internal/boltdb/paste_test.go new file mode 100644 index 0000000..bba0d33 --- /dev/null +++ b/internal/boltdb/paste_test.go @@ -0,0 +1,38 @@ +package boltdb + +import "testing" + +func TestValidatePasteKey(t *testing.T) { + tests := []struct { + key string + wantErr bool + errKind error + }{ + {"xd42__", false, nil}, + {"xd42_*", true, ErrKeyInvalidChar}, + {"xd42_/", true, ErrKeyInvalidChar}, + {"xd42_=", true, ErrKeyInvalidChar}, + {"xd42_", true, ErrKeyInvalidLength}, + {"xd42", true, ErrKeyInvalidLength}, + {"xd4", true, ErrKeyInvalidLength}, + {"xd", true, ErrKeyInvalidLength}, + {"x", true, ErrKeyInvalidLength}, + {"", true, ErrKeyInvalidLength}, + + {"KoJ5", false, nil}, + + {"__dGSJIIbBpr-SD0", false, nil}, + {"__dGSJIIbBpr-SD", true, ErrKeyInvalidLength}, + } + for _, tt := range tests { + t.Run(tt.key, func(t *testing.T) { + err := ValidatePasteKey(tt.key) + if (err != nil) != tt.wantErr { + t.Errorf("ValidatePasteKey() got error = %v, want error %v", err != nil, tt.wantErr) + } + if (err != nil) && err != tt.errKind { + t.Errorf("ValidatePasteKey() error = %v, want errKind %v", err, tt.errKind) + } + }) + } +} diff --git a/internal/db/db.go b/internal/db/db.go index 14dc2cd..025aa9d 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,204 +1,74 @@ package db import ( - "bytes" - "fmt" - "io" "log" - "net/http" "os" "time" "github.com/pkg/errors" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) - bolt "go.etcd.io/bbolt" - - gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" +const ( + // EnvDatabaseDriver is the key for the 'database driver' environment variable. + EnvDatabaseDriver = "RUSHLINK_DATABASE_DRIVER" + // EnvDatabasePath is the key for the 'database path' environment variable. + EnvDatabasePath = "RUSHLINK_DATABASE_PATH" + // EnvPostgresURL is the key for the 'postgresql URL' environment variable. + EnvPostgresURL = "RUSHLINK_POSTGRES_URL" + // EnvFileStorePath is the key for the environment variable locating the file store. + EnvFileStorePath = "RUSHLINK_FILE_STORE_PATH" ) // Database is the main rushlink database type. // -// Open a database using DB.Open() and close it in the end using DB.Close(). +// Open a database using OpenDBFromEnvironment(). Closing is not necessary. // Only one instance of DB should exist in a program at any moment. -type Database struct { - Bolt *bolt.DB -} +type Database = gorm.DB -// CurrentMigrateVersion holds the current "migrate version". -// -// If we alter the database format, we bump this number and write a new -// database migration in migrate(). -const CurrentMigrateVersion = 3 +var ( + gormLogger logger.Interface = logger.New(log.New(os.Stdout, "\r\n", log.LstdFlags), logger.Config{ + SlowThreshold: 50 * time.Millisecond, + LogLevel: logger.Warn, + Colorful: true, + }) + gormConfig = gorm.Config{Logger: gormLogger, PrepareStmt: true} +) -// BucketConf holds the name for the "configuration" bucket. -// -// This bucket holds the database version, secret site-wide keys, etc. -const BucketConf = "conf" +// OpenDBFromEnvironment tries to open an SQL database, described by +func OpenDBFromEnvironment() (*Database, error) { + const envNotSetMsg = "%v environment variable is not set" -// BucketPastes holds the name for the pastes bucket. -const BucketPastes = "pastes" - -// BucketFileUpload holds the name for the file-upload bucket. -const BucketFileUpload = "fileUpload" - -// KeyMigrateVersion stores the current migration version. If this value is less than -// CurrentMigrateVersion, the database has to be migrated. -const KeyMigrateVersion = "migrate_version" - -// OpenDB opens a database file located at path. -func OpenDB(path string, fs *FileStore) (*Database, error) { - if path == "" { - return nil, errors.New("database not set") + driver, prs := os.LookupEnv(EnvDatabaseDriver) + if !prs { + return nil, errors.Errorf(envNotSetMsg, EnvDatabaseDriver) } - - db, err := bolt.Open(path, 0660, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - return nil, errors.Wrapf(err, "failed to open database at '%v'", path) - } - if err := db.Update(func(tx *bolt.Tx) error { return migrate(tx, fs) }); err != nil { - return nil, err - } - return &Database{db}, nil -} - -// Close the bolt database -func (db *Database) Close() error { - if db == nil { - panic("no open database") - } - return db.Bolt.Close() -} - -// Initialize and migrate the database to the current version -func migrate(tx *bolt.Tx, fs *FileStore) error { - // Guidelines for error handling: - // - Errors based on malformed *structure* should be fatal! - // - Errors based on malformed *data* should print a warning - // (and if possible try to fix the error). - - dbVersion, err := dbVersion(tx) - if err != nil { - return err - } - - // Migrate the database to version 1 - if dbVersion < 1 { - log.Println("migrating database to version 1") - // Create conf bucket - _, err := tx.CreateBucket([]byte(BucketConf)) + switch driver { + case "sqlite": + path, prs := os.LookupEnv(EnvDatabasePath) + if !prs { + return nil, errors.Errorf(envNotSetMsg, EnvDatabasePath) + } + db, err := gorm.Open(sqlite.Open(path), &gormConfig) if err != nil { - return err + return nil, err } - // Create paste bucket - _, err = tx.CreateBucket([]byte(BucketPastes)) + return db, nil + case "postgres": + dsn, prs := os.LookupEnv(EnvPostgresURL) + if !prs { + return nil, errors.Errorf(envNotSetMsg, EnvPostgresURL) + } + db, err := gorm.Open(postgres.Open(dsn), &gormConfig) if err != nil { - return err - } - // Update the version number - if err := setDBVersion(tx, 1); err != nil { - return err + return nil, err } + return db, nil + default: + return nil, errors.Errorf("RUSHLINK_DATABASE_DRIVER should be either 'sqlite' or 'postgres' (not '%v'), "+ + "for more info see ", driver) } - - if dbVersion < 2 { - log.Println("migrating database to version 2") - // Create fileUpload bucket - _, err := tx.CreateBucket([]byte(BucketFileUpload)) - if err != nil { - return err - } - // Update the version number - if err := setDBVersion(tx, 2); err != nil { - return err - } - } - - if dbVersion < 3 { - log.Println("migrating database to version 3") - // In this version, we changed te way how Content-Types are being - // stored. Previously, we allowed clients to provide their own - // Content-Types for files, using the Content-Disposition header in - // multipart forms. The new way detects these types using - // http.DetectContentType. - // - // Scan through all the FileUploads and update their ContentTypes. - bucket := tx.Bucket([]byte(BucketFileUpload)) - cursor := bucket.Cursor() - var id, storedBytes []byte - id, storedBytes = cursor.First() - for id != nil { - fu, err := decodeFileUpload(storedBytes) - if err != nil { - log.Print("error: ", errors.Wrapf(err, "corrupted FileUpload in database at '%v'", id)) - id, storedBytes = cursor.Next() - continue - } - if fu.State != FileUploadStatePresent { - id, storedBytes = cursor.Next() - continue - } - filePath := fu.Path(fs) - file, err := os.Open(fu.Path(fs)) - if err != nil { - log.Print("error: ", errors.Wrapf(err, "could not open file at '%v'", filePath)) - id, storedBytes = cursor.Next() - continue - } - var buf bytes.Buffer - buf.Grow(512) - io.CopyN(&buf, file, 512) - contentType := http.DetectContentType(buf.Bytes()) - if contentType != fu.ContentType { - fu.ContentType = contentType - fu.Save(tx) - cursor.Seek(id) - } - id, storedBytes = cursor.Next() - } - // Update the version number - if err := setDBVersion(tx, 3); err != nil { - return err - } - } - - return nil -} - -// Get the current migrate version from the database -func dbVersion(tx *bolt.Tx) (int, error) { - conf := tx.Bucket([]byte(BucketConf)) - if conf == nil { - return 0, nil - } - dbVersionBytes := conf.Get([]byte(KeyMigrateVersion)) - if dbVersionBytes == nil { - return 0, nil - } - - // Version was already stored - var dbVersion int - if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil { - return 0, err - } - if dbVersion == 0 { - return 0, fmt.Errorf("database version is invalid (%v)", dbVersion) - } - if dbVersion > CurrentMigrateVersion { - return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CurrentMigrateVersion) - } - return dbVersion, nil -} - -// Update the current migrate version in the database -func setDBVersion(tx *bolt.Tx, version int) error { - conf, err := tx.CreateBucketIfNotExists([]byte(BucketConf)) - if err != nil { - return err - } - - versionBytes, err := gobmarsh.Marshal(version) - if err != nil { - return err - } - return conf.Put([]byte(KeyMigrateVersion), versionBytes) } diff --git a/internal/db/fileupload.go b/internal/db/fileupload.go index f390ef2..b15a66b 100644 --- a/internal/db/fileupload.go +++ b/internal/db/fileupload.go @@ -10,12 +10,11 @@ import ( "os" "path" "path/filepath" + "time" "github.com/google/uuid" "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" - - gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" + "gorm.io/gorm" ) // Use the Castagnoli checksum because of the acceleration on Intel CPUs @@ -31,11 +30,13 @@ type FileUploadState int // FileUpload models an uploaded file. type FileUpload struct { - // State of the FileUpload (present/deleted/etc). - State FileUploadState + ID uint `gorm:"primaryKey"` - // ID identifies this FileUpload. - ID uuid.UUID + // State of the FileUpload (present/deleted/etc). + State FileUploadState `gorm:"index"` + + // UUID publically identifies this FileUpload. + PubID uuid.UUID `gorm:"uniqueIndex"` // FileName contains the original filename of this FileUpload. FileName string @@ -48,6 +49,10 @@ type FileUpload struct { // This checksum is only meant to allow for the detection of random // database corruption. Checksum uint32 + + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt gorm.DeletedAt } const ( @@ -77,6 +82,19 @@ func (t FileUploadState) String() string { } } +// ErrFileUploadDoesNotExist occurs when a key does not exist in the database. +var ErrFileUploadDoesNotExist = errors.New("file not found in the database") + +// OpenFileStoreFromEnvironment tries to open a file store located at ${RUSHLINK_FILE_STORE_PATH}. +func OpenFileStoreFromEnvironment() (*FileStore, error) { + path, prs := os.LookupEnv(EnvFileStorePath) + if !prs { + err := errors.Errorf("%v environment variable is not set", EnvFileStorePath) + return nil, errors.Wrap(err, "opening file store") + } + return OpenFileStore(path) +} + // OpenFileStore opens the file storage at path. func OpenFileStore(path string) (*FileStore, error) { if path == "" { @@ -97,11 +115,11 @@ func (fs *FileStore) Path() string { } // filePath resolves the path of a file in the FileStore given some id and filename. -func (fs *FileStore) filePath(id uuid.UUID, fileName string) string { +func (fs *FileStore) filePath(pubID uuid.UUID, fileName string) string { if fs.path == "" { panic("fileStoreDir called while the file store path has not been set") } - return path.Join(fs.path, hex.EncodeToString(id[:]), fileName) + return path.Join(fs.path, hex.EncodeToString(pubID[:]), fileName) } // NewFileUpload creates a new FileUpload object. @@ -110,7 +128,7 @@ func (fs *FileStore) filePath(id uuid.UUID, fileName string) string { // `http.DetectContentType`. func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, error) { // Generate a file ID - id, err := uuid.NewRandom() + pubID, err := uuid.NewRandom() if err != nil { return nil, errors.Wrap(err, "generating UUID") } @@ -127,7 +145,7 @@ func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, er // Open the file on disk for writing baseName := filepath.Base(fileName) - filePath := fs.filePath(id, baseName) + filePath := fs.filePath(pubID, baseName) if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil { return nil, errors.Wrap(err, "creating file dir") } @@ -149,7 +167,7 @@ func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, er fu := &FileUpload{ State: FileUploadStatePresent, - ID: id, + PubID: pubID, FileName: baseName, ContentType: contentType, Checksum: hash.Sum32(), @@ -158,64 +176,33 @@ func NewFileUpload(fs *FileStore, r io.Reader, fileName string) (*FileUpload, er } // GetFileUpload tries to retrieve a FileUpload object from the bolt database. -func GetFileUpload(tx *bolt.Tx, id uuid.UUID) (*FileUpload, error) { - bucket := tx.Bucket([]byte(BucketFileUpload)) - if bucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload) +func GetFileUpload(db *gorm.DB, pubID uuid.UUID) (*FileUpload, error) { + var fus []FileUpload + if err := db.Unscoped().Limit(1).Where("pub_id = ?", pubID).Find(&fus).Error; err != nil { + return nil, err } - storedBytes := bucket.Get(id[:]) - if storedBytes == nil { - return nil, nil + if len(fus) == 0 { + return nil, ErrFileUploadDoesNotExist } - return decodeFileUpload(storedBytes) + return &fus[0], nil } // AllFileUploads tries to retrieve all FileUpload objects from the bolt database. -func AllFileUploads(tx *bolt.Tx) ([]FileUpload, error) { - bucket := tx.Bucket([]byte(BucketFileUpload)) - if bucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketFileUpload) - } +func AllFileUploads(db *gorm.DB) ([]FileUpload, error) { var fus []FileUpload - err := bucket.ForEach(func(_, storedBytes []byte) error { - fu, err := decodeFileUpload(storedBytes) - if err != nil { - return err - } - fus = append(fus, *fu) - return nil - }) - if err != nil { + if err := db.Find(&fus).Error; err != nil { return nil, err } return fus, nil } -func decodeFileUpload(storedBytes []byte) (*FileUpload, error) { - fu := &FileUpload{} - err := gobmarsh.Unmarshal(storedBytes, fu) - return fu, err -} - // Save saves a FileUpload in the database. -func (fu *FileUpload) Save(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(BucketFileUpload)) - if bucket == nil { - return errors.Errorf("bucket %v does not exist", BucketFileUpload) - } - - buf, err := gobmarsh.Marshal(fu) - if err != nil { - return errors.Wrap(err, "encoding for database failed") - } - if err := bucket.Put(fu.ID[:], buf); err != nil { - return errors.Wrap(err, "database transaction failed") - } - return nil +func (fu *FileUpload) Save(db *gorm.DB) error { + return db.Save(fu).Error } // Delete deletes a FileUpload from the database. -func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error { +func (fu *FileUpload) Delete(db *gorm.DB, fs *FileStore) error { // Remove the file in the backend filePath := fu.Path(fs) if err := os.Remove(filePath); err != nil { @@ -223,10 +210,7 @@ func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error { } // Update the file in the server - if err := (&FileUpload{ - ID: fu.ID, - State: FileUploadStateDeleted, - }).Save(tx); err != nil { + if err := db.Delete(fu).Error; err != nil { return err } @@ -237,12 +221,12 @@ func (fu *FileUpload) Delete(tx *bolt.Tx, fs *FileStore) error { // Path returns the path to this FileUpload in the FileStore provided in fs. func (fu *FileUpload) Path(fs *FileStore) string { - return fs.filePath(fu.ID, fu.FileName) + return fs.filePath(fu.PubID, fu.FileName) } // URL returns the URL for the FileUpload. func (fu *FileUpload) URL() *url.URL { - rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName + rawurl := "/uploads/" + hex.EncodeToString(fu.PubID[:]) + "/" + fu.FileName urlParse, err := url.Parse(rawurl) if err != nil { panic("could not construct /uploads/ url") diff --git a/internal/db/migrate.go b/internal/db/migrate.go new file mode 100644 index 0000000..e1f8a1a --- /dev/null +++ b/internal/db/migrate.go @@ -0,0 +1,46 @@ +package db + +import ( + "time" + + gormigrate "github.com/go-gormigrate/gormigrate/v2" + "github.com/google/uuid" + "gorm.io/gorm" +) + +// Gormigrate returns a Gormigrate migrator for the database. +func Gormigrate(db *gorm.DB) *gormigrate.Gormigrate { + return gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{ + { + ID: "202010251337", + Migrate: func(tx *gorm.DB) error { + type FileUpload struct { + ID uint `gorm:"primaryKey"` + State FileUploadState `gorm:"index"` + PubID uuid.UUID `gorm:"uniqueIndex"` + FileName string + ContentType string + Checksum uint32 + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt gorm.DeletedAt + } + type Paste struct { + ID uint `gorm:"primaryKey"` + Type PasteType `gorm:"index"` + State PasteState `gorm:"index"` + Content []byte + Key string `gorm:"uniqueIndex"` + DeleteToken string + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt gorm.DeletedAt + } + return tx.AutoMigrate(&FileUpload{}, &Paste{}) + }, + Rollback: func(tx *gorm.DB) error { + return tx.Migrator().DropTable(&FileUpload{}, &Paste{}) + }, + }, + }) +} diff --git a/internal/db/paste.go b/internal/db/paste.go index 46708d2..91e3882 100644 --- a/internal/db/paste.go +++ b/internal/db/paste.go @@ -12,7 +12,7 @@ import ( gobmarsh "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" "github.com/google/uuid" "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" + "gorm.io/gorm" ) // PasteType describes the type of Paste (i.e. file, redirect, [...]). @@ -23,15 +23,18 @@ type PasteState int // Paste describes the main Paste model in the database. type Paste struct { - Type PasteType - State PasteState + ID uint `gorm:"primaryKey"` + Type PasteType `gorm:"index"` + State PasteState `gorm:"index"` Content []byte - Key string + Key string `gorm:"uniqueIndex"` DeleteToken string - TimeCreated time.Time + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt gorm.DeletedAt } -// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen +// ReservedPasteKeys keys are designated reserved, and will not be randomly chosen. var ReservedPasteKeys = []string{"xd42", "example"} // Note: we use iota here. That means removals of PasteType* are not allowed, @@ -73,7 +76,7 @@ func ErrHTTPStatusCode(err error) int { switch err { case nil: return 0 - case ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist: + case gorm.ErrRecordNotFound, ErrKeyInvalidChar, ErrKeyInvalidLength, ErrPasteDoesNotExist: return http.StatusNotFound } return http.StatusInternalServerError @@ -112,11 +115,11 @@ func (t PasteState) String() string { } // GetPaste retrieves a paste from the database. -func GetPaste(tx *bolt.Tx, key string) (*Paste, error) { +func GetPaste(db *gorm.DB, key string) (*Paste, error) { if err := ValidatePasteKey(key); err != nil { return nil, err } - return GetPasteNoValidate(tx, key) + return GetPasteNoValidate(db, key) } // ValidatePasteKey validates the format of the key that has @@ -144,16 +147,15 @@ func ValidatePasteKey(key string) error { // GetPasteNoValidate retrieves a paste from the database without validating // the key format first. -func GetPasteNoValidate(tx *bolt.Tx, key string) (*Paste, error) { - pastesBucket := tx.Bucket([]byte(BucketPastes)) - if pastesBucket == nil { - return nil, errors.Errorf("bucket %v does not exist", BucketPastes) +func GetPasteNoValidate(db *gorm.DB, key string) (*Paste, error) { + var ps []Paste + if err := db.Unscoped().Limit(1).Where("key = ?", key).Find(&ps).Error; err != nil { + return nil, err } - storedBytes := pastesBucket.Get([]byte(key)) - if storedBytes == nil { + if len(ps) == 0 { return nil, ErrPasteDoesNotExist } - return decodePaste(storedBytes) + return &ps[0], nil } func decodePaste(storedBytes []byte) (*Paste, error) { @@ -163,44 +165,36 @@ func decodePaste(storedBytes []byte) (*Paste, error) { } // Save saves this Paste to the database. -func (p *Paste) Save(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte(BucketPastes)) - if bucket == nil { - return errors.Errorf("bucket %v does not exist", BucketPastes) - } - - buf, err := gobmarsh.Marshal(p) - if err != nil { - return errors.Wrap(err, "encoding for database failed") - } - if err := bucket.Put([]byte(p.Key), buf); err != nil { - return errors.Wrap(err, "database transaction failed") - } - return nil +func (p *Paste) Save(db *gorm.DB) error { + return db.Save(p).Error } // Delete deletes this Paste from the database. -func (p *Paste) Delete(tx *bolt.Tx, fs *FileStore) error { +func (p *Paste) Delete(db *gorm.DB, fs *FileStore) error { // Remove the (maybe) attached file if p.Type == PasteTypeFileUpload { fuID, err := uuid.FromBytes(p.Content) if err != nil { return errors.Wrap(err, "failed to parse uuid") } - fu, err := GetFileUpload(tx, fuID) + fu, err := GetFileUpload(db, fuID) if err != nil { return errors.Wrap(err, "failed to find file in database") } - if err := fu.Delete(tx, fs); err != nil { + if err := fu.Delete(db, fs); err != nil { return errors.Wrap(err, "failed to remove file") } } - // Replace the old paste with a new empty paste + // Wipe the old paste p.Type = PasteTypeUndef p.State = PasteStateDeleted p.Content = []byte{} - if err := p.Save(tx); err != nil { + if err := db.Save(&p).Error; err != nil { + return errors.Wrap(err, "failed to wipe paste in database") + } + // Soft-delete the paste as well + if err := db.Delete(&p).Error; err != nil { return errors.Wrap(err, "failed to delete paste in database") } return nil @@ -228,12 +222,7 @@ func (p *Paste) RedirectURL() *url.URL { // keys stored in the url-shorten database. // In tx, a Bolt transaction is given. Use minimumEntropy to set the mimimum // guessing entropy of the generated key. -func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) { - pastesBucket := tx.Bucket([]byte(BucketPastes)) - if pastesBucket == nil { - return "", errors.Errorf("bucket %v does not exist", BucketPastes) - } - +func GeneratePasteKey(db *gorm.DB, minimumEntropy int) (string, error) { epoch := 0 var key string for { @@ -243,10 +232,12 @@ func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) { return "", errors.Wrap(err, "url-key generation failed") } - found := pastesBucket.Get([]byte(key)) - if found == nil { - break + var count int64 + db.Unscoped().Model(&Paste{}).Where("key = ?", []byte(key)).Count(&count) + if err != nil { + return "", errors.Wrap(err, "failed to check if key already exists") } + alreadyInUse := count != 0 isReserved := false for _, reservedKey := range ReservedPasteKeys { @@ -255,10 +246,9 @@ func GeneratePasteKey(tx *bolt.Tx, minimumEntropy int) (string, error) { break } } - if !isReserved { + if !alreadyInUse && !isReserved { break } - epoch++ } return key, nil diff --git a/internal/db/paste_test.go b/internal/db/paste_test.go index 7d66aa2..59ca227 100644 --- a/internal/db/paste_test.go +++ b/internal/db/paste_test.go @@ -1,6 +1,24 @@ package db -import "testing" +import ( + "testing" + "testing/quick" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func OpenTemporaryDB() (*Database, error) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gormConfig) + if err != nil { + return nil, err + } + migrate := Gormigrate(db) + if err := migrate.Migrate(); err != nil { + return nil, err + } + return db, nil +} func TestValidatePasteKey(t *testing.T) { tests := []struct { @@ -36,3 +54,47 @@ func TestValidatePasteKey(t *testing.T) { }) } } + +func TestGeneratedKeysAreValid(t *testing.T) { + db, err := OpenTemporaryDB() + if err != nil { + t.Error(err) + } + var minimumEntropy int + checkGeneratedKeyValid := func() bool { + key, err := GeneratePasteKey(db, minimumEntropy) + if err != nil { + return false + } + return ValidatePasteKey(key) == nil + } + + for minimumEntropy = 0; minimumEntropy <= 80; minimumEntropy++ { + if err = quick.Check(checkGeneratedKeyValid, nil); err != nil { + t.Error(err) + } + } +} + +func TestTruncatedKeysAreInvalid(t *testing.T) { + db, err := OpenTemporaryDB() + if err != nil { + t.Error(err) + } + var minimumEntropy int + checkTruncatedKeysInvalid := func(truncRand uint) bool { + key, err := GeneratePasteKey(db, minimumEntropy) + if err != nil { + return false + } + trunc := int(truncRand % uint(len(key)-1)) + key = key[:trunc] + return ValidatePasteKey(key) == ErrKeyInvalidLength + } + + for minimumEntropy = 0; minimumEntropy <= 80; minimumEntropy++ { + if err = quick.Check(checkTruncatedKeysInvalid, nil); err != nil { + t.Error(err) + } + } +} diff --git a/metrics.go b/metrics.go index 35d3e4f..2c58732 100644 --- a/metrics.go +++ b/metrics.go @@ -6,12 +6,10 @@ import ( "time" "gitea.hashru.nl/dsprenkels/rushlink/internal/db" - "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - bolt "go.etcd.io/bbolt" ) const metricNamespace = "rushlink" @@ -25,12 +23,12 @@ var metricRequestsTotalCounter = prometheus.NewCounterVec(prometheus.CounterOpts func metricURLsTotal(database *db.Database) float64 { var metric float64 - if err := database.Bolt.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte("pastes")) - if bucket == nil { - return errors.New("bucket 'pastes' could not be found") + if err := database.Transaction(func(tx *db.Database) error { + var count int64 + if err := database.Model(&db.Paste{}).Count(&count).Error; err != nil { + return err } - metric = float64(bucket.Stats().KeyN) + metric = float64(count) return nil }); err != nil { log.Printf("error: %v", errors.Wrap(err, "fetching pastes_total metric"))