forked from electricdusk/rushlink
Use a more standardized project layout
In #2, mrngm made the point that we should move to a more standardized project structure. This commit does exactly that. The new project structure is based on the repository listed at <https://github.com/golang-standards/project-layout>. Fixes #2.
This commit is contained in:
Daan Sprenkels
112
internal/db/db.go
Normal file
112
internal/db/db.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
)
|
||||
|
||||
var DB *bolt.DB
|
||||
|
||||
// The current database version
|
||||
//
|
||||
// If we alter the database format, we bump this number and write a new
|
||||
// database migration in migrateDatabase().
|
||||
const CURRENT_MIGRATE_VERSION = 1
|
||||
|
||||
// Bucket storing everything that is not a bulk value. This includes stuff like
|
||||
// the database version, secret site-wide keys.
|
||||
const BUCKET_CONF = "conf"
|
||||
|
||||
// The main bucket for paste values and URL redirects
|
||||
const BUCKET_PASTES = "pastes"
|
||||
|
||||
// This value stores the current migration version. If this value is less than
|
||||
// CURRENT_MIGRATE_VERSION, the database has to be migrated.
|
||||
const KEY_MIGRATE_VERSION = "migrate_version"
|
||||
|
||||
// Open the bolt database
|
||||
func Init(name string) error {
|
||||
var err error
|
||||
DB, err = bolt.Open(name, 0666, &bolt.Options{Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to open database at '%v'", name)
|
||||
}
|
||||
return DB.Update(migrateDatabase)
|
||||
}
|
||||
|
||||
// Initialize and migrate the database to the current version
|
||||
func migrateDatabase(tx *bolt.Tx) error {
|
||||
dbVersion, err := dbVersion(tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrate the database to version 1
|
||||
if dbVersion < 1 {
|
||||
log.Println("migrating database to version 1")
|
||||
|
||||
// Create conf bucket
|
||||
_, err := tx.CreateBucket([]byte(BUCKET_CONF))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create paste bucket
|
||||
_, err = tx.CreateBucket([]byte(BUCKET_PASTES))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update the version number
|
||||
if err := setDBVersion(tx, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the current migrate version from the database
|
||||
func dbVersion(tx *bolt.Tx) (int, error) {
|
||||
conf := tx.Bucket([]byte(BUCKET_CONF))
|
||||
if conf == nil {
|
||||
return 0, nil
|
||||
}
|
||||
dbVersionBytes := conf.Get([]byte(KEY_MIGRATE_VERSION))
|
||||
if dbVersionBytes == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Version was already stored
|
||||
var dbVersion int
|
||||
if err := gobmarsh.Unmarshal(dbVersionBytes, &dbVersion); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if dbVersion == 0 {
|
||||
return 0, fmt.Errorf("database version is invalid (%v)", dbVersion)
|
||||
}
|
||||
if dbVersion > CURRENT_MIGRATE_VERSION {
|
||||
return 0, fmt.Errorf("database version is too recent (%v > %v)", dbVersion, CURRENT_MIGRATE_VERSION)
|
||||
}
|
||||
return dbVersion, nil
|
||||
}
|
||||
|
||||
// Update the current migrate version in the database
|
||||
func setDBVersion(tx *bolt.Tx, version int) error {
|
||||
conf, err := tx.CreateBucketIfNotExists([]byte(BUCKET_CONF))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
versionBytes, err := gobmarsh.Marshal(version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conf.Put([]byte(KEY_MIGRATE_VERSION), versionBytes)
|
||||
}
|
||||
395
internal/handlers/handlers.go
Normal file
395
internal/handlers/handlers.go
Normal file
@@ -0,0 +1,395 @@
|
||||
package handlers
|
||||
|
||||
//go:generate go get github.com/go-bindata/go-bindata
|
||||
//go:generate go get -u github.com/go-bindata/go-bindata/...
|
||||
//go:generate go-bindata -pkg $GOPACKAGE -prefix ../../assets ../../assets/...
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/pkg/errors"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
|
||||
"gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh"
|
||||
)
|
||||
|
||||
type PasteType int
|
||||
type PasteState int
|
||||
|
||||
type StoredPaste struct {
|
||||
Type PasteType
|
||||
State PasteState
|
||||
Content []byte
|
||||
Key []byte
|
||||
OwnerToken [16]byte
|
||||
TimeCreated time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
TypePaste PasteType = iota
|
||||
TypeRedirect
|
||||
)
|
||||
|
||||
const (
|
||||
StatePresent PasteState = iota
|
||||
StateDeleted
|
||||
)
|
||||
|
||||
const CookieOwnerToken = "owner_token"
|
||||
|
||||
// These keys are designated reserved, and will not be randomly chosen
|
||||
var ReservedPasteKeys [][]byte = [][]byte{[]byte("xd42"), []byte("example")}
|
||||
|
||||
// Base64 encoding and decoding
|
||||
var base64Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
|
||||
var base64Encoder = base64.NewEncoding(base64Alphabet).WithPadding(base64.NoPadding)
|
||||
|
||||
// Page contents
|
||||
var baseTemplate = template.New("empty")
|
||||
var indexTemplate = template.Must(baseTemplate.Parse(string(MustAsset("text/index.txt"))))
|
||||
|
||||
func (t PasteType) String() (string, error) {
|
||||
switch t {
|
||||
case TypePaste:
|
||||
return "paste", nil
|
||||
case TypeRedirect:
|
||||
return "redirect", nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid PasteType (%v)", t)
|
||||
}
|
||||
}
|
||||
|
||||
func (t PasteState) String() (string, error) {
|
||||
switch t {
|
||||
case StatePresent:
|
||||
return "present", nil
|
||||
case StateDeleted:
|
||||
return "deleted", nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid PasteState (%v)", t)
|
||||
}
|
||||
}
|
||||
|
||||
func IndexGetHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if err := indexTemplate.Execute(w, nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func IndexPostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(50 * 1000 * 1000); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
fmt.Fprintf(w, "Internal server error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Determine what kind of post this is, currently only `shorten=...`
|
||||
if len(r.PostForm) == 0 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
var buf []byte
|
||||
r.Body.Read(buf)
|
||||
io.WriteString(w, "empty body in POST request\n")
|
||||
return
|
||||
}
|
||||
shorten_values, prs := r.PostForm["shorten"]
|
||||
if !prs {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
io.WriteString(w, "no 'shorten' param supplied\n")
|
||||
return
|
||||
}
|
||||
if len(shorten_values) != 1 {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
io.WriteString(w, "only one 'shorten' param is allowed per request\n")
|
||||
return
|
||||
}
|
||||
|
||||
ShortenPostHandler(w, r)
|
||||
}
|
||||
|
||||
func PasteGetHandler(w http.ResponseWriter, r *http.Request) {
|
||||
pasteGetHandlerInner(w, r, false, false)
|
||||
}
|
||||
|
||||
func PasteGetHandlerNoRedirect(w http.ResponseWriter, r *http.Request) {
|
||||
pasteGetHandlerInner(w, r, true, false)
|
||||
}
|
||||
|
||||
func PasteGetHandlerMeta(w http.ResponseWriter, r *http.Request) {
|
||||
pasteGetHandlerInner(w, r, false, true)
|
||||
}
|
||||
|
||||
func pasteGetHandlerInner(w http.ResponseWriter, r *http.Request, noRedirect, showMeta bool) {
|
||||
vars := mux.Vars(r)
|
||||
key := vars["key"]
|
||||
var storedPaste *StoredPaste
|
||||
if err := db.DB.View(func(tx *bolt.Tx) error {
|
||||
var err error
|
||||
storedPaste, err = getURL(tx, []byte(key))
|
||||
return err
|
||||
}); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
log.Printf("error: %v\n", err)
|
||||
fmt.Fprintf(w, "internal server error: %v\n", err)
|
||||
return
|
||||
}
|
||||
if storedPaste == nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprintf(w, "url key not found in the database\n")
|
||||
return
|
||||
}
|
||||
|
||||
if showMeta {
|
||||
typeString, err := storedPaste.Type.String()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
log.Printf("error: %v\n", err)
|
||||
fmt.Fprintf(w, "internal server error: %v\n", err)
|
||||
return
|
||||
}
|
||||
stateString, err := storedPaste.State.String()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
log.Printf("error: %v\n", err)
|
||||
fmt.Fprintf(w, "internal server error: %v\n", err)
|
||||
return
|
||||
}
|
||||
isOwner := "no"
|
||||
ownerToken, ok := getOwnerTokenFromRequest(r)
|
||||
if ok && subtle.ConstantTimeCompare(ownerToken[:], storedPaste.OwnerToken[:]) == 1 {
|
||||
isOwner = "yes"
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "key: %v\n", string(storedPaste.Key))
|
||||
fmt.Fprintf(w, "type: %v\n", typeString)
|
||||
fmt.Fprintf(w, "state: %v\n", stateString)
|
||||
fmt.Fprintf(w, "created: %v\n", storedPaste.TimeCreated.String())
|
||||
fmt.Fprintf(w, "are you the owner: %v\n", isOwner)
|
||||
return
|
||||
}
|
||||
|
||||
switch storedPaste.State {
|
||||
case StatePresent:
|
||||
if !noRedirect {
|
||||
rawurl := string(storedPaste.Content)
|
||||
urlParse, err := url.Parse(rawurl)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
log.Printf("error: invalid URL ('%v') in database for key '%v': %v\n", rawurl, storedPaste.Key, err)
|
||||
fmt.Fprintf(w, "internal server error: invalid url in database\n")
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, urlParse.String(), http.StatusSeeOther)
|
||||
}
|
||||
w.Write(storedPaste.Content)
|
||||
case StateDeleted:
|
||||
w.WriteHeader(http.StatusGone)
|
||||
fmt.Fprintf(w, "key has been deleted\n")
|
||||
default:
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
log.Printf("error: invalid storedPaste.State (%v) for key '%v'\n", storedPaste.State, storedPaste.Key)
|
||||
fmt.Fprintf(w, "internal server error: invalid storedPaste.State (%v\n)", storedPaste.State)
|
||||
}
|
||||
}
|
||||
|
||||
func ShortenPostHandler(w http.ResponseWriter, r *http.Request) {
|
||||
rawurl := r.PostForm.Get("shorten")
|
||||
userURL, err := url.ParseRequestURI(rawurl)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "invalid url (%v): %v\n", err, rawurl)
|
||||
return
|
||||
}
|
||||
if userURL.Scheme == "" {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "invalid url (unspecified scheme)\n", rawurl)
|
||||
return
|
||||
}
|
||||
if userURL.Host == "" {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "invalid url (unspecified host)\n", rawurl)
|
||||
return
|
||||
}
|
||||
|
||||
var storedPaste *StoredPaste
|
||||
if err := db.DB.Update(func(tx *bolt.Tx) error {
|
||||
ownerKey, ok := getOwnerTokenFromRequest(r)
|
||||
if ok == false {
|
||||
// Owner key not supplied or invalid, generate a new one
|
||||
ownerKey, err = generateOwnerToken()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "generating OwnerToken")
|
||||
}
|
||||
}
|
||||
|
||||
sp, err := shortenURL(tx, userURL, ownerKey)
|
||||
storedPaste = sp
|
||||
return err
|
||||
}); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
log.Printf("error: %v\n", err)
|
||||
fmt.Fprintf(w, "internal server error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
saveURL, err := r.URL.Parse(string(storedPaste.Key))
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "parsing url")
|
||||
log.Printf("error: %v\n", err)
|
||||
fmt.Fprintf(w, "internal server error: %v\n", err)
|
||||
return
|
||||
}
|
||||
var base64OwnerToken = make([]byte, 24)
|
||||
base64Encoder.Encode(base64OwnerToken, storedPaste.OwnerToken[:])
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "URL saved at %v\n", saveURL)
|
||||
isNotPrint := func(r rune) bool { return !unicode.IsPrint(r) }
|
||||
fmt.Fprintf(w, "Owner key is %s\n", strings.TrimRightFunc(string(base64OwnerToken), isNotPrint))
|
||||
}
|
||||
|
||||
// Retrieve a URL from the database
|
||||
func getURL(tx *bolt.Tx, key []byte) (*StoredPaste, error) {
|
||||
shortenBucket := tx.Bucket([]byte(db.BUCKET_PASTES))
|
||||
if shortenBucket == nil {
|
||||
return nil, fmt.Errorf("bucket %v does not exist", db.BUCKET_PASTES)
|
||||
}
|
||||
storedBytes := shortenBucket.Get(key)
|
||||
if storedBytes == nil {
|
||||
return nil, nil
|
||||
}
|
||||
storedPaste := &StoredPaste{}
|
||||
err := gobmarsh.Unmarshal(storedBytes, storedPaste)
|
||||
return storedPaste, err
|
||||
}
|
||||
|
||||
// Add a new URL to the database
|
||||
//
|
||||
// Returns the new ID if the url was successfully shortened
|
||||
func shortenURL(tx *bolt.Tx, userURL *url.URL, ownerKey [16]byte) (*StoredPaste, error) {
|
||||
shortenBucket := tx.Bucket([]byte(db.BUCKET_PASTES))
|
||||
if shortenBucket == nil {
|
||||
return nil, fmt.Errorf("bucket %v does not exist", db.BUCKET_PASTES)
|
||||
}
|
||||
|
||||
// Generate a key until it is not in the database, this occurs in O(log N),
|
||||
// where N is the amount of keys stored in the url-shorten database.
|
||||
epoch := 0
|
||||
var urlKey []byte
|
||||
for {
|
||||
var err error
|
||||
urlKey, err = generateURLKey(epoch)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "url-key generation failed")
|
||||
}
|
||||
|
||||
found := shortenBucket.Get(urlKey)
|
||||
if found == nil {
|
||||
break
|
||||
}
|
||||
|
||||
isReserved := false
|
||||
for _, reservedKey := range ReservedPasteKeys {
|
||||
if bytes.HasPrefix(urlKey, reservedKey) {
|
||||
isReserved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !isReserved {
|
||||
break
|
||||
}
|
||||
|
||||
epoch++
|
||||
}
|
||||
|
||||
// Store the new key
|
||||
storedPaste := StoredPaste{
|
||||
Type: TypeRedirect,
|
||||
State: StatePresent,
|
||||
Content: []byte(userURL.String()),
|
||||
Key: urlKey,
|
||||
OwnerToken: ownerKey,
|
||||
TimeCreated: time.Now().UTC(),
|
||||
}
|
||||
storedBytes, err := gobmarsh.Marshal(storedPaste)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "encoding for database failed")
|
||||
}
|
||||
if err := shortenBucket.Put(urlKey, storedBytes); err != nil {
|
||||
return nil, errors.Wrap(err, "database transaction failed")
|
||||
}
|
||||
return &storedPaste, nil
|
||||
}
|
||||
|
||||
func generateURLKey(epoch int) ([]byte, error) {
|
||||
urlKey := make([]byte, 4+epoch)
|
||||
_, err := rand.Read(urlKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Put all the values in the range 0..64 for easier base64-encoding
|
||||
for i := 0; i < len(urlKey); i++ {
|
||||
urlKey[i] &= 0x3F
|
||||
}
|
||||
// Implement truncate-resistance by forcing the prefix to
|
||||
// 0b111110xxxxxxxxxx
|
||||
// ^----- {epoch} ones followed by a single 0
|
||||
//
|
||||
// Example when epoch is 1: prefix is 0b10.
|
||||
i := 0
|
||||
for i < epoch {
|
||||
// Set this bit to 1
|
||||
limb := i / 6
|
||||
bit := i % 6
|
||||
urlKey[limb] |= 1 << uint(5-bit)
|
||||
i++
|
||||
}
|
||||
// Finally set the next bit to 0
|
||||
limb := i / 6
|
||||
bit := i % 6
|
||||
urlKey[limb] &= ^(1 << uint(5-bit))
|
||||
|
||||
// Convert this ID to a canonical base64 notation
|
||||
for i := range urlKey {
|
||||
urlKey[i] = base64Alphabet[urlKey[i]]
|
||||
}
|
||||
return urlKey, nil
|
||||
}
|
||||
|
||||
func generateOwnerToken() ([16]byte, error) {
|
||||
var ownerKey [16]byte
|
||||
_, err := rand.Read(ownerKey[:])
|
||||
if err != nil {
|
||||
return ownerKey, err
|
||||
}
|
||||
return ownerKey, nil
|
||||
}
|
||||
|
||||
func getOwnerTokenFromRequest(r *http.Request) ([16]byte, bool) {
|
||||
var ownerKey [16]byte
|
||||
ownerKeyCookie, err := r.Cookie(CookieOwnerToken)
|
||||
if err != nil && err != http.ErrNoCookie {
|
||||
return ownerKey, false
|
||||
}
|
||||
if ownerKeyCookie != nil {
|
||||
n, err := base64Encoder.Strict().Decode(ownerKey[:], []byte(ownerKeyCookie.Value))
|
||||
if err == nil || n == 16 {
|
||||
return ownerKey, true
|
||||
}
|
||||
}
|
||||
return ownerKey, false
|
||||
}
|
||||
55
internal/metrics/metrics.go
Normal file
55
internal/metrics/metrics.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
|
||||
"gitea.hashru.nl/dsprenkels/rushlink/internal/db"
|
||||
)
|
||||
|
||||
const (
|
||||
METRICS_ADDR = "127.0.0.1:58614"
|
||||
)
|
||||
|
||||
func StartMetricsServer() {
|
||||
var (
|
||||
_ = promauto.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Namespace: "rushlink",
|
||||
Subsystem: "shorten",
|
||||
Name: "urls_total",
|
||||
Help: "The current amount of shortened urls in the database.",
|
||||
}, func() float64 {
|
||||
var metric float64
|
||||
if err := db.DB.View(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte("shorten"))
|
||||
if bucket == nil {
|
||||
return errors.New("bucket 'shorten' could not be found")
|
||||
}
|
||||
metric = float64(bucket.Stats().KeyN)
|
||||
return nil
|
||||
}); err != nil {
|
||||
log.Printf("error: %v", errors.Wrap(err, "fetching shorten_urls_total metric"))
|
||||
return 0
|
||||
}
|
||||
return metric
|
||||
})
|
||||
)
|
||||
|
||||
router := mux.NewRouter()
|
||||
router.Handle("/metrics", promhttp.Handler()).Methods("GET")
|
||||
srv := &http.Server{
|
||||
Handler: router,
|
||||
Addr: METRICS_ADDR,
|
||||
WriteTimeout: 15 * time.Second,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
}
|
||||
log.Fatal(srv.ListenAndServe())
|
||||
}
|
||||
Reference in New Issue
Block a user