diff --git a/assets/templates/html/newFileUploadPasteSuccess.html.tmpl b/assets/templates/html/newFileUploadPasteSuccess.html.tmpl new file mode 100644 index 0000000..334f2cb --- /dev/null +++ b/assets/templates/html/newFileUploadPasteSuccess.html.tmpl @@ -0,0 +1,3 @@ +{{define "title"}} +Success - rushlink +{{end}} \ No newline at end of file diff --git a/assets/templates/txt/newFileUploadPasteSuccess.txt.tmpl b/assets/templates/txt/newFileUploadPasteSuccess.txt.tmpl new file mode 100644 index 0000000..bbceb50 --- /dev/null +++ b/assets/templates/txt/newFileUploadPasteSuccess.txt.tmpl @@ -0,0 +1,4 @@ +https://{{.Request.Host}}/{{.Paste.Key}} +{{if .Request.PostForm.deleteToken -}} +https://{{.Request.Host}}/{{.Paste.Key}}?deleteToken={{.Paste.DeleteToken | urlquery}} +{{end -}} diff --git a/cmd/rushlink/main.go b/cmd/rushlink/main.go index b83a937..9a6dbad 100644 --- a/cmd/rushlink/main.go +++ b/cmd/rushlink/main.go @@ -8,18 +8,22 @@ import ( ) var ( - databasePath = flag.String("database", "", "location of the database file") - httpListen = flag.String("listen", "127.0.0.1:8000", "listen address (host:port)") + databasePath = flag.String("database", "", "location of the database file") + fileStorePath = flag.String("file-store", "", "path to the directory where uploaded files will be stored") + httpListen = flag.String("listen", "127.0.0.1:8000", "listen address (host:port)") metricsListen = flag.String("metrics_listen", "127.0.0.1:58614", "listen address for metrics (host:port)") ) func main() { flag.Parse() - if err := rushlink.Open(*databasePath); err != nil { + if err := rushlink.OpenDB(*databasePath); err != nil { + log.Fatalln(err) + } + defer rushlink.CloseDB() + if err := rushlink.OpenFileStore(*fileStorePath); err != nil { log.Fatalln(err) } - defer rushlink.Close() go rushlink.StartMetricsServer(*metricsListen) rushlink.StartMainServer(*httpListen) diff --git a/db.go b/db.go index b749011..315160e 100644 --- a/db.go +++ b/db.go @@ -15,7 +15,7 @@ var DB *bolt.DB // // If we alter the database format, we bump this number and write a new // database migration in migrateDatabase(). -const CURRENT_MIGRATE_VERSION = 1 +const CURRENT_MIGRATE_VERSION = 2 // Bucket storing everything that is not a bulk value. This includes stuff like // the database version, secret site-wide keys. @@ -24,12 +24,15 @@ const BUCKET_CONF = "conf" // The main bucket for paste values and URL redirects const BUCKET_PASTES = "pastes" +// The main bucket for file uploads +const BUCKET_FILE_UPLOAD = "fileUpload" + // This value stores the current migration version. If this value is less than // CURRENT_MIGRATE_VERSION, the database has to be migrated. const KEY_MIGRATE_VERSION = "migrate_version" // Open the bolt database -func Open(path string) error { +func OpenDB(path string) error { if path == "" { return errors.New("database not set") } @@ -43,7 +46,7 @@ func Open(path string) error { } // Close the bolt database -func Close() error { +func CloseDB() error { if DB == nil { panic("no open database") } @@ -60,25 +63,35 @@ func migrateDatabase(tx *bolt.Tx) error { // Migrate the database to version 1 if dbVersion < 1 { log.Println("migrating database to version 1") - // Create conf bucket _, err := tx.CreateBucket([]byte(BUCKET_CONF)) if err != nil { return err } - // Create paste bucket _, err = tx.CreateBucket([]byte(BUCKET_PASTES)) if err != nil { return err } - // Update the version number if err := setDBVersion(tx, 1); err != nil { return err } } + if dbVersion < 2 { + log.Println("migrating database to version 2") + // Create fileUpload bucket + _, err := tx.CreateBucket([]byte(BUCKET_FILE_UPLOAD)) + if err != nil { + return err + } + // Update the version number + if err := setDBVersion(tx, 2); err != nil { + return err + } + } + return nil } diff --git a/fileupload.go b/fileupload.go new file mode 100644 index 0000000..bcef1e8 --- /dev/null +++ b/fileupload.go @@ -0,0 +1,182 @@ +package rushlink + +import ( + "encoding/hex" + "hash/crc32" + "io" + "log" + "net/http" + "net/url" + "os" + "path" + + "github.com/google/uuid" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" +) + +// Use the Castagnoli checksum because of the acceleration on Intel CPUs +var checksumTable = crc32.MakeTable(crc32.Castagnoli) + +// Where to store the uploaded files +var fileStoreDir = "" + +// Custom HTTP filesystem handler +type fileUploadFileSystem struct { + fs http.FileSystem +} + +// Open opens file +func (fs fileUploadFileSystem) Open(path string) (http.File, error) { + log.Println(path) + file, err := fs.fs.Open(path) + if err != nil { + return nil, errors.Wrap(err, "opening file") + } + stat, err := file.Stat() + if err != nil { + return nil, errors.Wrap(err, "file.Stat()") + } + if stat.IsDir() { + return nil, errors.New("directory index not allowed") + } + return file, nil +} + +type fileUploadState int + +type fileUpload struct { + State fileUploadState + ID uuid.UUID + FileName string + ContentType string + Checksum uint32 +} + +const ( + dirMode os.FileMode = 0750 + fileMode os.FileMode = 0640 +) + +const ( + fileUploadStateUndef fileUploadState = 0 + fileUploadStatePresent = 1 + fileUploadStateDeleted = 2 +) + +func (t fileUploadState) String() string { + switch t { + case fileUploadStateUndef: + return "unknown" + case fileUploadStatePresent: + return "present" + case fileUploadStateDeleted: + return "deleted" + default: + return "invalid" + } +} + +func OpenFileStore(path string) error { + if path == "" { + return errors.New("file store path not set") + } + + // Try to create the file store directory if it does not yet exist + if err := os.MkdirAll(path, dirMode); err != nil { + return errors.Wrap(err, "creating file store directory") + } + + fileStoreDir = path[:] + return nil +} + +func newFileUpload(tx *bolt.Tx, r io.Reader, fileName string, contentType string) (*fileUpload, error) { + id, err := uuid.NewRandom() + if err != nil { + return nil, errors.Wrap(err, "generating UUID") + } + + filePath := fileStorePath(id, fileName) + if err := os.Mkdir(path.Dir(filePath), dirMode); err != nil { + return nil, errors.Wrap(err, "creating file dir") + } + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, fileMode) + if err != nil { + return nil, errors.Wrap(err, "opening file") + } + defer file.Close() + + hash := crc32.New(checksumTable) + tee := io.TeeReader(r, hash) + _, err = io.Copy(file, tee) + if err != nil { + return nil, errors.Wrap(err, "writing to file") + } + + fu := &fileUpload{ + State: fileUploadStatePresent, + ID: id, + FileName: fileName, + ContentType: contentType, + Checksum: hash.Sum32(), + } + if err := fu.save(tx); err != nil { + return nil, err + } + return fu, nil +} + +func getFileUpload(tx *bolt.Tx, id uuid.UUID) (*fileUpload, error) { + bucket := tx.Bucket([]byte(BUCKET_FILE_UPLOAD)) + if bucket == nil { + return nil, errors.Errorf("bucket %v does not exist", BUCKET_FILE_UPLOAD) + } + storedBytes := bucket.Get(id[:]) + if storedBytes == nil { + return nil, nil + } + fu := &fileUpload{} + err := Unmarshal(storedBytes, fu) + return fu, err +} + +func (fu *fileUpload) save(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BUCKET_FILE_UPLOAD)) + if bucket == nil { + return errors.Errorf("bucket %v does not exist", BUCKET_FILE_UPLOAD) + } + + buf, err := Marshal(fu) + if err != nil { + return errors.Wrap(err, "encoding for database failed") + } + if err := bucket.Put(fu.ID[:], buf); err != nil { + return errors.Wrap(err, "database transaction failed") + } + return nil +} + +func (fu *fileUpload) delete(tx *bolt.Tx) error { + // Replace the old paste with a new empty paste + return (&fileUpload{ + ID: fu.ID, + State: fileUploadStateDeleted, + }).save(tx) +} + +func (fu *fileUpload) url() *url.URL { + rawurl := "/uploads/" + hex.EncodeToString(fu.ID[:]) + "/" + fu.FileName + urlParse, err := url.Parse(rawurl) + if err != nil { + panic("could not construct /uploads/ url") + } + return urlParse +} + +func fileStorePath(id uuid.UUID, fileName string) string { + if fileStoreDir == "" { + panic("fileStoreDir called while the file store path has not been set") + } + return path.Join(fileStoreDir, hex.EncodeToString(id[:]), fileName) +} diff --git a/go.mod b/go.mod index 5ffbfc3..2430008 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module gitea.hashru.nl/dsprenkels/rushlink go 1.12 require ( + github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.3 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.1.0 diff --git a/go.sum b/go.sum index 3556000..420f99d 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/handlers.go b/handlers.go index dfb9a17..6de1de3 100644 --- a/handlers.go +++ b/handlers.go @@ -4,11 +4,15 @@ import ( "crypto/subtle" "encoding/base64" "fmt" + "io" "log" + "mime/multipart" "net/http" "net/url" + "os" "time" + "github.com/google/uuid" "github.com/gorilla/mux" "github.com/pkg/errors" bolt "go.etcd.io/bbolt" @@ -31,36 +35,48 @@ var ReservedPasteKeys = []string{"xd42", "example"} var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" var base64Encoder = base64.RawURLEncoding.WithPadding(base64.NoPadding) -func (t pasteType) String() string { - switch t { - case typeUndef: - return "unknown" - case typePaste: - return "paste" - case typeRedirect: - return "redirect" - default: - return "invalid" - } -} - -func (t pasteState) String() string { - switch t { - case stateUndef: - return "unknown" - case statePresent: - return "present" - case stateDeleted: - return "deleted" - default: - return "invalid" - } -} - func indexGetHandler(w http.ResponseWriter, r *http.Request) { render(w, r, "index", map[string]interface{}{}) } +func uploadFileGetHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + id := vars["id"] + + var fu *fileUpload + var badID bool + if err := DB.View(func(tx *bolt.Tx) error { + fuID, err := uuid.Parse(id) + if err != nil { + badID = true + return err + } + fu, err = getFileUpload(tx, fuID) + return err + }); err != nil { + if badID { + renderError(w, r, http.StatusNotFound, "malformed file id") + return + } else { + panic(err) + } + } + + filePath := fileStorePath(fu.ID, fu.FileName) + file, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + log.Printf("error: %v should exist according to the database, but it doesn't", filePath) + renderError(w, r, http.StatusNotFound, "file not found") + return + } else { + panic(err) + } + } + w.Header().Set("Content-Type", fu.ContentType) + io.Copy(w, file) +} + func viewPasteHandler(w http.ResponseWriter, r *http.Request) { viewPasteHandlerInner(w, r, 0) } @@ -77,14 +93,28 @@ func viewPasteHandlerInner(w http.ResponseWriter, r *http.Request, flags viewPas vars := mux.Vars(r) key := vars["key"] var p *paste + var fuID *uuid.UUID + var fu *fileUpload if err := DB.View(func(tx *bolt.Tx) error { var err error p, err = getPaste(tx, key) - return err + if err != nil { + return err + } + if p != nil && p.Type == pasteTypeFileUpload { + var id uuid.UUID + copy(id[:], p.Content) + fuID = &id + fu, err = getFileUpload(tx, id) + if err != nil { + return err + } + } + return nil }); err != nil { panic(err) - return } + if p == nil { renderError(w, r, http.StatusNotFound, "url key not found in the database") return @@ -116,47 +146,101 @@ func viewPasteHandlerInner(w http.ResponseWriter, r *http.Request, flags viewPas } switch p.State { - case statePresent: - if flags&viewNoRedirect == 0 { - rawurl := string(p.Content) - urlParse, err := url.Parse(rawurl) - if err != nil { - panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key)) + case pasteStatePresent: + var location string + switch p.Type { + case pasteTypeFileUpload: + if fu == nil { + panic(fmt.Sprintf("file for id %v does not exist in database\n", fuID)) } - http.Redirect(w, r, urlParse.String(), http.StatusSeeOther) + location = fu.url().String() + break + case pasteTypeRedirect: + location = p.redirectURL().String() + break + default: + panic("paste type unsupported") } - w.Write(p.Content) - case stateDeleted: - renderError(w, r, http.StatusGone, "paste has been deleted") + if flags&viewNoRedirect == 0 { + http.Redirect(w, r, location, http.StatusSeeOther) + } + fmt.Fprint(w, location) + case pasteStateDeleted: + renderError(w, r, http.StatusGone, "paste has been deleted\n") default: panic(errors.Errorf("invalid paste.State (%v) for key '%v'", p.State, p.Key)) } } func newPasteHandler(w http.ResponseWriter, r *http.Request) { - if err := r.ParseMultipartForm(50 * 1000 * 1000); err != nil { - panic(err) - } + newPasteHandlerMultipart(w, r, func(w http.ResponseWriter, r *http.Request) { + newPasteHandlerURLEncoded(w, r, func(w http.ResponseWriter, r *http.Request) { + renderError(w, r, http.StatusBadRequest, "no form data in request\n") + }) + }) +} - // Determine what kind of post this is, currently only `shorten=...` - if len(r.PostForm) == 0 { - renderError(w, r, http.StatusBadRequest, "empty body in POST request\n") +// Try to parse the new-paste request as multi-part form. +// +// If there is no multi-part form in the request, this handler will call next. +func newPasteHandlerMultipart(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + reader, err := r.MultipartReader() + if err != nil { + next(w, r) return } - shorten_values, prs := r.PostForm["shorten"] - if !prs { + + part, err := reader.NextPart() + if err != nil { + if err == io.EOF { + renderError(w, r, http.StatusBadRequest, "multipart form is empty\n") + return + } + panic(errors.Wrap(err, "multipart.Reader.NextPart")) + } + if part.FormName() != "file" { + msg := fmt.Sprintf("invalid multipart form name: %v", part.FormName()) + renderError(w, r, http.StatusBadRequest, msg) + return + } + + newFileUploadPasteHandler(w, r, *part) +} + +func newFileUploadPasteHandler(w http.ResponseWriter, r *http.Request, part multipart.Part) { + var fu *fileUpload + var paste *paste + if err := DB.Update(func(tx *bolt.Tx) error { + var err error + // Create the fileUpload in the database + fu, err = newFileUpload(tx, &part, part.FileName(), part.Header.Get("Content-Type")) + if err != nil { + panic(errors.Wrap(err, "creating fileUpload")) + } + + paste, err = shortenFileUploadID(tx, fu.ID) + return err + }); err != nil { + panic(err) + } + data := map[string]interface{}{"Paste": paste} + render(w, r, "newFileUploadPasteSuccess", data) +} + +func newPasteHandlerURLEncoded(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { + if err := r.ParseForm(); err != nil { + next(w, r) + return + } + shorten := r.PostFormValue("shorten") + if shorten == "" { renderError(w, r, http.StatusBadRequest, "no 'shorten' param given\n") return } - if len(shorten_values) != 1 { - renderError(w, r, http.StatusBadRequest, "only one 'shorten' param is allowed per request\n") - return - } - - newRedirectPasteHandler(w, r) + newRedirectPasteHandler(w, r, shorten) } -func newRedirectPasteHandler(w http.ResponseWriter, r *http.Request) { +func newRedirectPasteHandler(w http.ResponseWriter, r *http.Request, shorten string) { rawurl := r.PostForm.Get("shorten") userURL, err := url.ParseRequestURI(rawurl) if err != nil { @@ -165,11 +249,11 @@ func newRedirectPasteHandler(w http.ResponseWriter, r *http.Request) { return } if userURL.Scheme == "" { - renderError(w, r, http.StatusBadRequest, "invalid url (unspecified scheme)") + renderError(w, r, http.StatusBadRequest, "invalid url (unspecified scheme)\n") return } if userURL.Host == "" { - renderError(w, r, http.StatusBadRequest, "invalid url (unspecified host)") + renderError(w, r, http.StatusBadRequest, "invalid url (unspecified host)\n") return } @@ -193,7 +277,7 @@ func deletePasteHandler(w http.ResponseWriter, r *http.Request) { deleteToken := getDeleteTokenFromRequest(r) if deleteToken == "" { - renderError(w, r, http.StatusBadRequest, "no delete token provided") + renderError(w, r, http.StatusBadRequest, "no delete token provided\n") return } @@ -211,15 +295,29 @@ func deletePasteHandler(w http.ResponseWriter, r *http.Request) { return errors.New("invalid delete token") }); err != nil { log.Printf("error: %v\n", err) - renderError(w, r, errorCode, fmt.Sprintf("error: %v", err)) + renderError(w, r, errorCode, fmt.Sprintf("error: %v\n", err)) return } } +// Add a new fileUpload redirect to the database +// +// Returns the new paste key if the fileUpload was successfully added to the +// database +func shortenFileUploadID(tx *bolt.Tx, id uuid.UUID) (*paste, error) { + return shorten(tx, pasteTypeFileUpload, id[:]) +} + // Add a new URL to the database // -// Returns the new ID if the url was successfully shortened +// Returns the new paste key if the url was successfully shortened func shortenURL(tx *bolt.Tx, userURL *url.URL) (*paste, error) { + return shorten(tx, pasteTypeRedirect, []byte(userURL.String())) +} + +// Add a paste (of any kind) to the database with arbitrary content. +func shorten(tx *bolt.Tx, ty pasteType, content []byte) (*paste, error) { + // Generate the paste key pasteKey, err := generatePasteKey(tx) if err != nil { return nil, errors.Wrap(err, "generating paste key") @@ -233,9 +331,9 @@ func shortenURL(tx *bolt.Tx, userURL *url.URL) (*paste, error) { // Store the new key p := paste{ - Type: typeRedirect, - State: statePresent, - Content: []byte(userURL.String()), + Type: ty, + State: pasteStatePresent, + Content: content, Key: pasteKey, DeleteToken: deleteToken, TimeCreated: time.Now().UTC(), diff --git a/paste.go b/paste.go index e885b76..28c6fef 100644 --- a/paste.go +++ b/paste.go @@ -3,6 +3,7 @@ package rushlink import ( "crypto/rand" "encoding/hex" + "net/url" "strings" "time" @@ -23,17 +24,44 @@ type paste struct { } const ( - typeUndef pasteType = 0 - typePaste = 1 - typeRedirect = 2 + pasteTypeUndef pasteType = 0 + pasteTypePaste = 1 + pasteTypeRedirect = 2 + pasteTypeFileUpload = 3 ) const ( - stateUndef pasteState = 0 - statePresent = 1 - stateDeleted = 2 + pasteStateUndef pasteState = 0 + pasteStatePresent = 1 + pasteStateDeleted = 2 ) +func (t pasteType) String() string { + switch t { + case pasteTypeUndef: + return "unknown" + case pasteTypePaste: + return "paste" + case pasteTypeRedirect: + return "redirect" + default: + return "invalid" + } +} + +func (t pasteState) String() string { + switch t { + case pasteStateUndef: + return "unknown" + case pasteStatePresent: + return "present" + case pasteStateDeleted: + return "deleted" + default: + return "invalid" + } +} + // Retrieve a paste from the database func getPaste(tx *bolt.Tx, key string) (*paste, error) { pastesBucket := tx.Bucket([]byte(BUCKET_PASTES)) @@ -65,15 +93,31 @@ func (p *paste) save(tx *bolt.Tx) error { return nil } -func (p paste) delete(tx *bolt.Tx) error { +func (p *paste) delete(tx *bolt.Tx) error { // Replace the old paste with a new empty paste return (&paste{ Key: p.Key, - State: stateDeleted, + State: pasteStateDeleted, DeleteToken: p.DeleteToken, }).save(tx) } +// Get the URL from this paste. +// +// This function assumes that the paste is valid. If the paste struct is +// corrupted in some way, this function will panic. +func (p *paste) redirectURL() *url.URL { + if p.Type != pasteTypeRedirect { + panic("expected p.Type to be pasteTypeRedirect") + } + rawurl := string(p.Content) + urlParse, err := url.Parse(rawurl) + if err != nil { + panic(errors.Wrapf(err, "invalid URL ('%v') in database for key '%v'", rawurl, p.Key)) + } + return urlParse +} + // Generate a key until it is not in the database, this occurs in O(log N), // where N is the amount of keys stored in the url-shorten database. func generatePasteKey(tx *bolt.Tx) (string, error) { diff --git a/router.go b/router.go index 1ea3361..71b28cb 100644 --- a/router.go +++ b/router.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "net/http" + "runtime/debug" "time" "github.com/gorilla/mux" @@ -15,12 +16,14 @@ func recoveryMiddleware(next http.Handler) http.Handler { defer func() { if err := recover(); err != nil { log.Printf("error: panic while recovering from another panic: %v\n", err) + debug.PrintStack() fmt.Fprintf(w, "internal server error: %v\n", err) } }() if err := recover(); err != nil { log.Printf("error: %v\n", err) + debug.PrintStack() renderInternalServerError(w, r, err) } }() @@ -39,6 +42,7 @@ func StartMainServer(addr string) { router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/meta", viewPasteHandlerMeta).Methods("GET") router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}", deletePasteHandler).Methods("DELETE") router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/delete", deletePasteHandler).Methods("POST") + router.HandleFunc("/uploads/{id:[A-Za-z0-9-_]+}/{filename:.+}", uploadFileGetHandler).Methods("GET") srv := &http.Server{ Handler: router,