electricdusk/rushlink
3
3
Fork 3
Code Issues 21 Pull Requests 2 Releases Activity

Add support for file uploads

Browse Source
This commit is contained in:
Daan Sprenkels 2019-11-10 19:03:57 +01:00
parent d0b8a9ab9e
commit 66eb0bf97a
10 changed files with 434 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
assets/templates/html/newFileUploadPasteSuccess.html.tmpl Normal file
View File

@ -0,0 +1,3 @@
{{define "title"}}
Success - rushlink
{{end}}

4
assets/templates/txt/newFileUploadPasteSuccess.txt.tmpl Normal file
View File

@ -0,0 +1,4 @@
https://{{.Request.Host}}/{{.Paste.Key}}
{{if .Request.PostForm.deleteToken -}}
https://{{.Request.Host}}/{{.Paste.Key}}?deleteToken={{.Paste.DeleteToken | urlquery}}
{{end -}}

8
cmd/rushlink/main.go
View File

@ -9,6 +9,7 @@ import (
var (
databasePath = flag.String("database", "", "location of the database file")
fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored")
httpListen = flag.String("listen", "127.0.0.1:8000", "listen address (host:port)")
metricsListen = flag.String("metrics_listen", "127.0.0.1:58614", "listen address for metrics (host:port)")
)
@ -16,10 +17,13 @@ var (
func main() {
flag.Parse()
if err := rushlink.Open(*databasePath); err != nil {
if err := rushlink.OpenDB(*databasePath); err != nil {
log.Fatalln(err)
}
defer rushlink.CloseDB()
if err := rushlink.OpenFileStore(*fileStorePath); err != nil {
log.Fatalln(err)
}
defer rushlink.Close()
go rushlink.StartMetricsServer(*metricsListen)
rushlink.StartMainServer(*httpListen)

25
db.go
View File

@ -15,7 +15,7 @@ var DB *bolt.DB
//
// If we alter the database format, we bump this number and write a new
// database migration in migrateDatabase().
const CURRENT_MIGRATE_VERSION = 1
const CURRENT_MIGRATE_VERSION = 2
// Bucket storing everything that is not a bulk value. This includes stuff like
// the database version, secret site-wide keys.
@ -24,12 +24,15 @@ const BUCKET_CONF = "conf"
// The main bucket for paste values and URL redirects
const BUCKET_PASTES = "pastes"
// The main bucket for file uploads
const BUCKET_FILE_UPLOAD = "fileUpload"
// This value stores the current migration version. If this value is less than
// CURRENT_MIGRATE_VERSION, the database has to be migrated.
const KEY_MIGRATE_VERSION = "migrate_version"
// Open the bolt database
func Open(path string) error {
func OpenDB(path string) error {
if path == "" {
return errors.New("database not set")
}
@ -43,7 +46,7 @@ func Open(path string) error {
}
// Close the bolt database
func Close() error {
func CloseDB() error {
if DB == nil {
panic("no open database")
}
@ -60,25 +63,35 @@ func migrateDatabase(tx *bolt.Tx) error {
// Migrate the database to version 1
if dbVersion < 1 {
log.Println("migrating database to version 1")
// Create conf bucket
_, err := tx.CreateBucket([]byte(BUCKET_CONF))
if err != nil {
return err
}
// Create paste bucket
_, err = tx.CreateBucket([]byte(BUCKET_PASTES))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 1); err != nil {
return err
}
}
if dbVersion < 2 {
log.Println("migrating database to version 2")
// Create fileUpload bucket
_, err := tx.CreateBucket([]byte(BUCKET_FILE_UPLOAD))
if err != nil {
return err
}
// Update the version number
if err := setDBVersion(tx, 2); err != nil {
return err
}
}
return nil
}

182
fileupload.go Normal file
View File

@ -0,0 +1,182 @@
package rushlink
import (
"encoding/hex"
"hash/crc32"
"io"
"log"
"net/http"
"net/url"
"os"
"path"
"github.com/google/uuid"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
// Use the Castagnoli checksum because of the acceleration on Intel CPUs
var checksumTable = crc32.MakeTable(crc32.Castagnoli)
// Where to store the uploaded files
var fileStoreDir = ""
// Custom HTTP filesystem handler
type fileUploadFileSystem struct {
fs http.FileSystem
}
// Open opens file
func (fs fileUploadFileSystem) Open(path string) (http.File, error) {
log.Println(path)
file, err := fs.fs.Open(path)
if err != nil {
return nil, errors.Wrap(err, "opening file")
}
stat, err := file.Stat()
if err != nil {
return nil, errors.Wrap(err, "file.Stat()")
}
if stat.IsDir() {
return nil, errors.New("directory index not allowed")
}
return file, nil
}
type fileUploadState int
type fileUpload struct {
State fileUploadState
ID uuid.UUID
FileName string
ContentType string
Checksum uint32
}
const (
dirMode os.FileMode = 0750
fileMode os.FileMode = 0640
)
const (
fileUploadStateUndef fileUploadState = 0
fileUploadStatePresent = 1
fileUploadStateDeleted = 2
)
func (t fileUploadState) String() string {
switch t {
case fileUploadStateUndef:
return "unknown"
case fileUploadStatePresent:
return "present"
case fileUploadStateDeleted:
return "deleted"
default:
return "invalid"
}
}
func OpenFileStore(path string) error {
if path == "" {
return errors.New("file store path not set")
}
// Try to create the file store directory if it does not yet exist
if err := os.MkdirAll(path, dirMode); err != nil {
return errors.Wrap(err, "creating file store directory")
}
fileStoreDir = path[:]
return nil
}
func newFileUpload(tx *bolt.Tx, r io.Reader, fileName string, contentType string) (*fileUpload, error) {
id, err := uuid.NewRandom()
if err != nil {
return nil, errors.Wrap(err, "generating UUID")
}
filePath := fileStorePath(id, fileName)
if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil {
return nil, errors.Wrap(err, "creating file dir")
}
file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode)
if err != nil {
return nil, errors.Wrap(err, "opening file")
}
defer file.Close()
hash := crc32.New(checksumTable)
tee := io.TeeReader(r, hash)
_, err = io.Copy(file, tee)
if err != nil {
return nil, errors.Wrap(err, "writing to file")
}
fu := &fileUpload{
State: fileUploadStatePresent,
ID: id,
FileName: fileName,
ContentType: contentType,
Checksum: hash.Sum32(),
}
if err := fu.save(tx); err != nil {
return nil, err
}
return fu, nil
}
func getFileUpload(tx *bolt.Tx, id uuid.UUID) (*fileUpload, error) {
bucket := tx.Bucket([]byte(BUCKET_FILE_UPLOAD))
if bucket == nil {
return nil, errors.Errorf("bucket %v does not exist", BUCKET_FILE_UPLOAD)
}
storedBytes := bucket.Get(id[:])
if storedBytes == nil {
return nil, nil
}
fu := &fileUpload{}
err := Unmarshal(storedBytes, fu)
return fu, err
}
func (fu *fileUpload) save(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(BUCKET_FILE_UPLOAD))
if bucket == nil {
return errors.Errorf("bucket %v does not exist", BUCKET_FILE_UPLOAD)
}
buf, err := Marshal(fu)
if err != nil {
return errors.Wrap(err, "encoding for database failed")
}
if err := bucket.Put(fu.ID[:], buf); err != nil {
return errors.Wrap(err, "database transaction failed")
}
return nil
}
func (fu *fileUpload) delete(tx *bolt.Tx) error {
// Replace the old paste with a new empty paste
return (&fileUpload{
ID: fu.ID,
State: fileUploadStateDeleted,
}).save(tx)
}
func (fu *fileUpload) url() *url.URL {
rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName
urlParse, err := url.Parse(rawurl)
if err != nil {
panic("could not construct /uploads/ url")
}
return urlParse
}
func fileStorePath(id uuid.UUID, fileName string) string {
if fileStoreDir == "" {
panic("fileStoreDir called while the file store path has not been set")
}
return path.Join(fileStoreDir, hex.EncodeToString(id[:]), fileName)
}

1
go.mod
View File

@ -3,6 +3,7 @@ module gitea.hashru.nl/dsprenkels/rushlink
go 1.12
require (
github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.7.3
github.com/pkg/errors v0.8.1
github.com/prometheus/client_golang v1.1.0

2
go.sum
View File

@ -17,6 +17,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=

216
handlers.go
View File

@ -4,11 +4,15 @@ import (
"crypto/subtle"
"encoding/base64"
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"net/url"
"os"
"time"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
@ -31,36 +35,48 @@ var ReservedPasteKeys = []string{"xd42", "example"}
var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding)
func (t pasteType) String() string {
switch t {
case typeUndef:
return "unknown"
case typePaste:
return "paste"
case typeRedirect:
return "redirect"
default:
return "invalid"
}
}
func (t pasteState) String() string {
switch t {
case stateUndef:
return "unknown"
case statePresent:
return "present"
case stateDeleted:
return "deleted"
default:
return "invalid"
}
}
func indexGetHandler(w http.ResponseWriter, r *http.Request) {
render(w, r, "index", map[string]interface{}{})
}
func uploadFileGetHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["id"]
var fu *fileUpload
var badID bool
if err := DB.View(func(tx *bolt.Tx) error {
fuID, err := uuid.Parse(id)
if err != nil {
badID = true
return err
}
fu, err = getFileUpload(tx, fuID)
return err
}); err != nil {
if badID {
renderError(w, r, http.StatusNotFound, "malformed file id")
return
} else {
panic(err)
}
}
filePath := fileStorePath(fu.ID, fu.FileName)
file, err := os.Open(filePath)
if err != nil {
if os.IsNotExist(err) {
log.Printf("error: %v should exist according to the database, but it doesn't", filePath)
renderError(w, r, http.StatusNotFound, "file not found")
return
} else {
panic(err)
}
}
w.Header().Set("Content-Type", fu.ContentType)
io.Copy(w, file)
}
func viewPasteHandler(w http.ResponseWriter, r *http.Request) {
viewPasteHandlerInner(w, r, 0)
}
@ -77,14 +93,28 @@ func viewPasteHandlerInner(w http.ResponseWriter, r *http.Request, flags viewPas
vars := mux.Vars(r)
key := vars["key"]
var p *paste
var fuID *uuid.UUID
var fu *fileUpload
if err := DB.View(func(tx *bolt.Tx) error {
var err error
p, err = getPaste(tx, key)
if err != nil {
return err
}
if p != nil && p.Type == pasteTypeFileUpload {
var id uuid.UUID
copy(id[:], p.Content)
fuID = &id
fu, err = getFileUpload(tx, id)
if err != nil {
return err
}
}
return nil
}); err != nil {
panic(err)
return
}
if p == nil {
renderError(w, r, http.StatusNotFound, "url key not found in the database")
return
@ -116,47 +146,101 @@ func viewPasteHandlerInner(w http.ResponseWriter, r *http.Request, flags viewPas
}
switch p.State {
case statePresent:
case pasteStatePresent:
var location string
switch p.Type {
case pasteTypeFileUpload:
if fu == nil {
panic(fmt.Sprintf("file for id %v does not exist in database\n", fuID))
}
location = fu.url().String()
break
case pasteTypeRedirect:
location = p.redirectURL().String()
break
default:
panic("paste type unsupported")
}
if flags&viewNoRedirect == 0 {
rawurl := string(p.Content)
urlParse, err := url.Parse(rawurl)
if err != nil {
panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key))
http.Redirect(w, r, location, http.StatusSeeOther)
}
http.Redirect(w, r, urlParse.String(), http.StatusSeeOther)
}
w.Write(p.Content)
case stateDeleted:
renderError(w, r, http.StatusGone, "paste has been deleted")
fmt.Fprint(w, location)
case pasteStateDeleted:
renderError(w, r, http.StatusGone, "paste has been deleted\n")
default:
panic(errors.Errorf("invalid paste.State (%v) for key '%v'", p.State, p.Key))
}
}
func newPasteHandler(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(50 * 1000 * 1000); err != nil {
panic(err)
newPasteHandlerMultipart(w, r, func(w http.ResponseWriter, r *http.Request) {
newPasteHandlerURLEncoded(w, r, func(w http.ResponseWriter, r *http.Request) {
renderError(w, r, http.StatusBadRequest, "no form data in request\n")
})
})
}
// Determine what kind of post this is, currently only `shorten=...`
if len(r.PostForm) == 0 {
renderError(w, r, http.StatusBadRequest, "empty body in POST request\n")
// Try to parse the new-paste request as multi-part form.
//
// If there is no multi-part form in the request, this handler will call next.
func newPasteHandlerMultipart(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
reader, err := r.MultipartReader()
if err != nil {
next(w, r)
return
}
shorten_values, prs := r.PostForm["shorten"]
if !prs {
part, err := reader.NextPart()
if err != nil {
if err == io.EOF {
renderError(w, r, http.StatusBadRequest, "multipart form is empty\n")
return
}
panic(errors.Wrap(err, "multipart.Reader.NextPart"))
}
if part.FormName() != "file" {
msg := fmt.Sprintf("invalid multipart form name: %v", part.FormName())
renderError(w, r, http.StatusBadRequest, msg)
return
}
newFileUploadPasteHandler(w, r, *part)
}
func newFileUploadPasteHandler(w http.ResponseWriter, r *http.Request, part multipart.Part) {
var fu *fileUpload
var paste *paste
if err := DB.Update(func(tx *bolt.Tx) error {
var err error
// Create the fileUpload in the database
fu, err = newFileUpload(tx, &part, part.FileName(), part.Header.Get("Content-Type"))
if err != nil {
panic(errors.Wrap(err, "creating fileUpload"))
}
paste, err = shortenFileUploadID(tx, fu.ID)
return err
}); err != nil {
panic(err)
}
data := map[string]interface{}{"Paste": paste}
render(w, r, "newFileUploadPasteSuccess", data)
}
func newPasteHandlerURLEncoded(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
if err := r.ParseForm(); err != nil {
next(w, r)
return
}
shorten := r.PostFormValue("shorten")
if shorten == "" {
renderError(w, r, http.StatusBadRequest, "no 'shorten' param given\n")
return
}
if len(shorten_values) != 1 {
renderError(w, r, http.StatusBadRequest, "only one 'shorten' param is allowed per request\n")
return
newRedirectPasteHandler(w, r, shorten)
}
newRedirectPasteHandler(w, r)
}
func newRedirectPasteHandler(w http.ResponseWriter, r *http.Request) {
func newRedirectPasteHandler(w http.ResponseWriter, r *http.Request, shorten string) {
rawurl := r.PostForm.Get("shorten")
userURL, err := url.ParseRequestURI(rawurl)
if err != nil {
@ -165,11 +249,11 @@ func newRedirectPasteHandler(w http.ResponseWriter, r *http.Request) {
return
}
if userURL.Scheme == "" {
renderError(w, r, http.StatusBadRequest, "invalid url (unspecified scheme)")
renderError(w, r, http.StatusBadRequest, "invalid url (unspecified scheme)\n")
return
}
if userURL.Host == "" {
renderError(w, r, http.StatusBadRequest, "invalid url (unspecified host)")
renderError(w, r, http.StatusBadRequest, "invalid url (unspecified host)\n")
return
}
@ -193,7 +277,7 @@ func deletePasteHandler(w http.ResponseWriter, r *http.Request) {
deleteToken := getDeleteTokenFromRequest(r)
if deleteToken == "" {
renderError(w, r, http.StatusBadRequest, "no delete token provided")
renderError(w, r, http.StatusBadRequest, "no delete token provided\n")
return
}
@ -211,15 +295,29 @@ func deletePasteHandler(w http.ResponseWriter, r *http.Request) {
return errors.New("invalid delete token")
}); err != nil {
log.Printf("error: %v\n", err)
renderError(w, r, errorCode, fmt.Sprintf("error: %v", err))
renderError(w, r, errorCode, fmt.Sprintf("error: %v\n", err))
return
}
}
// Add a new fileUpload redirect to the database
//
// Returns the new paste key if the fileUpload was successfully added to the
// database
func shortenFileUploadID(tx *bolt.Tx, id uuid.UUID) (*paste, error) {
return shorten(tx, pasteTypeFileUpload, id[:])
}
// Add a new URL to the database
//
// Returns the new ID if the url was successfully shortened
// Returns the new paste key if the url was successfully shortened
func shortenURL(tx *bolt.Tx, userURL *url.URL) (*paste, error) {
return shorten(tx, pasteTypeRedirect, []byte(userURL.String()))
}
// Add a paste (of any kind) to the database with arbitrary content.
func shorten(tx *bolt.Tx, ty pasteType, content []byte) (*paste, error) {
// Generate the paste key
pasteKey, err := generatePasteKey(tx)
if err != nil {
return nil, errors.Wrap(err, "generating paste key")
@ -233,9 +331,9 @@ func shortenURL(tx *bolt.Tx, userURL *url.URL) (*paste, error) {
// Store the new key
p := paste{
Type: typeRedirect,
State: statePresent,
Content: []byte(userURL.String()),
Type: ty,
State: pasteStatePresent,
Content: content,
Key: pasteKey,
DeleteToken: deleteToken,
TimeCreated: time.Now().UTC(),

60
paste.go
View File

@ -3,6 +3,7 @@ package rushlink
import (
"crypto/rand"
"encoding/hex"
"net/url"
"strings"
"time"
@ -23,17 +24,44 @@ type paste struct {
}
const (
typeUndef pasteType = 0
typePaste = 1
typeRedirect = 2
pasteTypeUndef pasteType = 0
pasteTypePaste = 1
pasteTypeRedirect = 2
pasteTypeFileUpload = 3
)
const (
stateUndef pasteState = 0
statePresent = 1
stateDeleted = 2
pasteStateUndef pasteState = 0
pasteStatePresent = 1
pasteStateDeleted = 2
)
func (t pasteType) String() string {
switch t {
case pasteTypeUndef:
return "unknown"
case pasteTypePaste:
return "paste"
case pasteTypeRedirect:
return "redirect"
default:
return "invalid"
}
}
func (t pasteState) String() string {
switch t {
case pasteStateUndef:
return "unknown"
case pasteStatePresent:
return "present"
case pasteStateDeleted:
return "deleted"
default:
return "invalid"
}
}
// Retrieve a paste from the database
func getPaste(tx *bolt.Tx, key string) (*paste, error) {
pastesBucket := tx.Bucket([]byte(BUCKET_PASTES))
@ -65,15 +93,31 @@ func (p *paste) save(tx *bolt.Tx) error {
return nil
}
func (p paste) delete(tx *bolt.Tx) error {
func (p *paste) delete(tx *bolt.Tx) error {
// Replace the old paste with a new empty paste
return (&paste{
Key: p.Key,
State: stateDeleted,
State: pasteStateDeleted,
DeleteToken: p.DeleteToken,
}).save(tx)
}
// Get the URL from this paste.
//
// This function assumes that the paste is valid. If the paste struct is
// corrupted in some way, this function will panic.
func (p *paste) redirectURL() *url.URL {
if p.Type != pasteTypeRedirect {
panic("expected p.Type to be pasteTypeRedirect")
}
rawurl := string(p.Content)
urlParse, err := url.Parse(rawurl)
if err != nil {
panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key))
}
return urlParse
}
// Generate a key until it is not in the database, this occurs in O(log N),
// where N is the amount of keys stored in the url-shorten database.
func generatePasteKey(tx *bolt.Tx) (string, error) {

4
router.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"log"
"net/http"
"runtime/debug"
"time"
"github.com/gorilla/mux"
@ -15,12 +16,14 @@ func recoveryMiddleware(next http.Handler) http.Handler {
defer func() {
if err := recover(); err != nil {
log.Printf("error: panic while recovering from another panic: %v\n", err)
debug.PrintStack()
fmt.Fprintf(w, "internal server error: %v\n", err)
}
}()
if err := recover(); err != nil {
log.Printf("error: %v\n", err)
debug.PrintStack()
renderInternalServerError(w, r, err)
}
}()
@ -39,6 +42,7 @@ func StartMainServer(addr string) {
router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/meta", viewPasteHandlerMeta).Methods("GET")
router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}", deletePasteHandler).Methods("DELETE")
router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/delete", deletePasteHandler).Methods("POST")
router.HandleFunc("/uploads/{id:[A-Za-z0-9-_]+}/{filename:.+}", uploadFileGetHandler).Methods("GET")
srv := &http.Server{
Handler: router,