From 544c093c353257bc67395e2a085f1c20318a182c Mon Sep 17 00:00:00 2001 From: Daan Sprenkels Date: Tue, 10 Sep 2019 17:52:45 +0200 Subject: [PATCH] Use a more standardized project layout In #2, mrngm made the point that we should move to a more standardized project structure. This commit does exactly that. The new project structure is based on the repository listed at . Fixes #2. --- {text => assets/text}/index.txt | 0 cmd/rushlink/{rushlink.go => main.go} | 31 ++++------- cmd/rushlink/metrics.go | 54 ------------------ go.mod | 1 + go.sum | 1 + {cmd/rushlink => internal/db}/db.go | 19 ++++++- .../handlers}/handlers.go | 34 ++++++------ internal/metrics/metrics.go | 55 +++++++++++++++++++ {gobmarsh => pkg/gobmarsh}/marshal.go | 0 9 files changed, 103 insertions(+), 92 deletions(-) rename {text => assets/text}/index.txt (100%) rename cmd/rushlink/{rushlink.go => main.go} (54%) delete mode 100644 cmd/rushlink/metrics.go rename {cmd/rushlink => internal/db}/db.go (85%) rename {cmd/rushlink => internal/handlers}/handlers.go (90%) create mode 100644 internal/metrics/metrics.go rename {gobmarsh => pkg/gobmarsh}/marshal.go (100%) diff --git a/text/index.txt b/assets/text/index.txt similarity index 100% rename from text/index.txt rename to assets/text/index.txt diff --git a/cmd/rushlink/rushlink.go b/cmd/rushlink/main.go similarity index 54% rename from cmd/rushlink/rushlink.go rename to cmd/rushlink/main.go index 70c95af..6b3b459 100644 --- a/cmd/rushlink/rushlink.go +++ b/cmd/rushlink/main.go @@ -8,7 +8,10 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" + + "gitea.hashru.nl/dsprenkels/rushlink/internal/db" + "gitea.hashru.nl/dsprenkels/rushlink/internal/handlers" + "gitea.hashru.nl/dsprenkels/rushlink/internal/metrics" ) type ParsedArguments struct { @@ -16,7 +19,6 @@ type ParsedArguments struct { } var appConfig ParsedArguments -var db *bolt.DB func main() { // Parse the arguments and construct the ParsedArguments @@ -26,29 +28,18 @@ func main() { } appConfig = *appConfigRef - // Open the bolt database - db, err = bolt.Open(appConfig.databaseName, 0666, &bolt.Options{Timeout: 1 * time.Second}) - if err != nil { - decoratedErr := errors.Wrapf(err, "failed to open database at '%v'", appConfig.databaseName) - log.Fatal(decoratedErr) - } - defer db.Close() - err = db.Update(migrateDatabase) - if err != nil { - decoratedErr := errors.Wrapf(err, "failed to migrate database") - log.Fatal(decoratedErr) - } + db.Init(appConfig.databaseName) // Export prometheus metrics - go startMetricsServer() + go metrics.StartMetricsServer() // Initialize Gorilla router router := mux.NewRouter() - router.HandleFunc("/", indexGetHandler).Methods("GET") - router.HandleFunc("/", indexPostHandler).Methods("POST") - router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}", pasteGetHandler).Methods("GET") - router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/nr", pasteGetHandlerNoRedirect).Methods("GET") - router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/meta", pasteGetHandlerMeta).Methods("GET") + router.HandleFunc("/", handlers.IndexGetHandler).Methods("GET") + router.HandleFunc("/", handlers.IndexPostHandler).Methods("POST") + router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}", handlers.PasteGetHandler).Methods("GET") + router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/nr", handlers.PasteGetHandlerNoRedirect).Methods("GET") + router.HandleFunc("/{key:[A-Za-z0-9-_]{4,}}/meta", handlers.PasteGetHandlerMeta).Methods("GET") // Start the server srv := &http.Server{ diff --git a/cmd/rushlink/metrics.go b/cmd/rushlink/metrics.go deleted file mode 100644 index f1cb3f8..0000000 --- a/cmd/rushlink/metrics.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "log" - "net/http" - "time" - - bolt "go.etcd.io/bbolt" - - "github.com/gorilla/mux" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -const ( - METRICS_ADDR = "127.0.0.1:58614" -) - -var ( - shortenUrlsTotal = promauto.NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: "rushlink", - Subsystem: "shorten", - Name: "urls_total", - Help: "The current amount of shortened urls in the database.", - }, func() float64 { - var metric float64 - if err := db.View(func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte("shorten")) - if bucket == nil { - return errors.New("bucket 'shorten' could not be found") - } - metric = float64(bucket.Stats().KeyN) - return nil - }); err != nil { - log.Printf("error: %v", errors.Wrap(err, "fetching shorten_urls_total metric")) - return 0 - } - return metric - }) -) - -func startMetricsServer() { - router := mux.NewRouter() - router.Handle("/metrics", promhttp.Handler()).Methods("GET") - srv := &http.Server{ - Handler: router, - Addr: METRICS_ADDR, - WriteTimeout: 15 * time.Second, - ReadTimeout: 15 * time.Second, - } - log.Fatal(srv.ListenAndServe()) -} diff --git a/go.mod b/go.mod index 5ffbfc3..08f20a7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module gitea.hashru.nl/dsprenkels/rushlink go 1.12 require ( + github.com/go-bindata/go-bindata v3.1.2+incompatible // indirect github.com/gorilla/mux v1.7.3 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.1.0 diff --git a/go.sum b/go.sum index 3556000..0ad81ed 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= diff --git a/cmd/rushlink/db.go b/internal/db/db.go similarity index 85% rename from cmd/rushlink/db.go rename to internal/db/db.go index 38a54a7..2de10e1 100644 --- a/cmd/rushlink/db.go +++ b/internal/db/db.go @@ -1,13 +1,18 @@ -package main +package db import ( "fmt" "log" + "time" - "gitea.hashru.nl/dsprenkels/rushlink/gobmarsh" + "github.com/pkg/errors" bolt "go.etcd.io/bbolt" + + "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" ) +var DB *bolt.DB + // The current database version // // If we alter the database format, we bump this number and write a new @@ -25,6 +30,16 @@ const BUCKET_PASTES = "pastes" // CURRENT_MIGRATE_VERSION, the database has to be migrated. const KEY_MIGRATE_VERSION = "migrate_version" +// Open the bolt database +func Init(name string) error { + var err error + DB, err = bolt.Open(name, 0666, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return errors.Wrapf(err, "failed to open database at '%v'", name) + } + return DB.Update(migrateDatabase) +} + // Initialize and migrate the database to the current version func migrateDatabase(tx *bolt.Tx) error { dbVersion, err := dbVersion(tx) diff --git a/cmd/rushlink/handlers.go b/internal/handlers/handlers.go similarity index 90% rename from cmd/rushlink/handlers.go rename to internal/handlers/handlers.go index 8af4f38..39077ac 100644 --- a/cmd/rushlink/handlers.go +++ b/internal/handlers/handlers.go @@ -1,8 +1,8 @@ -package main +package handlers //go:generate go get github.com/go-bindata/go-bindata //go:generate go get -u github.com/go-bindata/go-bindata/... -//go:generate go-bindata -pkg $GOPACKAGE text/ +//go:generate go-bindata -pkg $GOPACKAGE -prefix ../../assets ../../assets/... import ( "bytes" @@ -19,10 +19,12 @@ import ( "time" "unicode" - "gitea.hashru.nl/dsprenkels/rushlink/gobmarsh" "github.com/gorilla/mux" "github.com/pkg/errors" bolt "go.etcd.io/bbolt" + + "gitea.hashru.nl/dsprenkels/rushlink/internal/db" + "gitea.hashru.nl/dsprenkels/rushlink/pkg/gobmarsh" ) type PasteType int @@ -82,13 +84,13 @@ func (t PasteState) String() (string, error) { } } -func indexGetHandler(w http.ResponseWriter, r *http.Request) { +func IndexGetHandler(w http.ResponseWriter, r *http.Request) { if err := indexTemplate.Execute(w, nil); err != nil { panic(err) } } -func indexPostHandler(w http.ResponseWriter, r *http.Request) { +func IndexPostHandler(w http.ResponseWriter, r *http.Request) { if err := r.ParseMultipartForm(50 * 1000 * 1000); err != nil { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "Internal server error: %v\n", err) @@ -115,18 +117,18 @@ func indexPostHandler(w http.ResponseWriter, r *http.Request) { return } - shortenPostHandler(w, r) + ShortenPostHandler(w, r) } -func pasteGetHandler(w http.ResponseWriter, r *http.Request) { +func PasteGetHandler(w http.ResponseWriter, r *http.Request) { pasteGetHandlerInner(w, r, false, false) } -func pasteGetHandlerNoRedirect(w http.ResponseWriter, r *http.Request) { +func PasteGetHandlerNoRedirect(w http.ResponseWriter, r *http.Request) { pasteGetHandlerInner(w, r, true, false) } -func pasteGetHandlerMeta(w http.ResponseWriter, r *http.Request) { +func PasteGetHandlerMeta(w http.ResponseWriter, r *http.Request) { pasteGetHandlerInner(w, r, false, true) } @@ -134,7 +136,7 @@ func pasteGetHandlerInner(w http.ResponseWriter, r *http.Request, noRedirect, sh vars := mux.Vars(r) key := vars["key"] var storedPaste *StoredPaste - if err := db.View(func(tx *bolt.Tx) error { + if err := db.DB.View(func(tx *bolt.Tx) error { var err error storedPaste, err = getURL(tx, []byte(key)) return err @@ -204,7 +206,7 @@ func pasteGetHandlerInner(w http.ResponseWriter, r *http.Request, noRedirect, sh } } -func shortenPostHandler(w http.ResponseWriter, r *http.Request) { +func ShortenPostHandler(w http.ResponseWriter, r *http.Request) { rawurl := r.PostForm.Get("shorten") userURL, err := url.ParseRequestURI(rawurl) if err != nil { @@ -224,7 +226,7 @@ func shortenPostHandler(w http.ResponseWriter, r *http.Request) { } var storedPaste *StoredPaste - if err := db.Update(func(tx *bolt.Tx) error { + if err := db.DB.Update(func(tx *bolt.Tx) error { ownerKey, ok := getOwnerTokenFromRequest(r) if ok == false { // Owner key not supplied or invalid, generate a new one @@ -262,9 +264,9 @@ func shortenPostHandler(w http.ResponseWriter, r *http.Request) { // Retrieve a URL from the database func getURL(tx *bolt.Tx, key []byte) (*StoredPaste, error) { - shortenBucket := tx.Bucket([]byte(BUCKET_PASTES)) + shortenBucket := tx.Bucket([]byte(db.BUCKET_PASTES)) if shortenBucket == nil { - return nil, fmt.Errorf("bucket %v does not exist", BUCKET_PASTES) + return nil, fmt.Errorf("bucket %v does not exist", db.BUCKET_PASTES) } storedBytes := shortenBucket.Get(key) if storedBytes == nil { @@ -279,9 +281,9 @@ func getURL(tx *bolt.Tx, key []byte) (*StoredPaste, error) { // // Returns the new ID if the url was successfully shortened func shortenURL(tx *bolt.Tx, userURL *url.URL, ownerKey [16]byte) (*StoredPaste, error) { - shortenBucket := tx.Bucket([]byte(BUCKET_PASTES)) + shortenBucket := tx.Bucket([]byte(db.BUCKET_PASTES)) if shortenBucket == nil { - return nil, fmt.Errorf("bucket %v does not exist", BUCKET_PASTES) + return nil, fmt.Errorf("bucket %v does not exist", db.BUCKET_PASTES) } // Generate a key until it is not in the database, this occurs in O(log N), diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..8ff53bb --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,55 @@ +package metrics + +import ( + "log" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + bolt "go.etcd.io/bbolt" + + "gitea.hashru.nl/dsprenkels/rushlink/internal/db" +) + +const ( + METRICS_ADDR = "127.0.0.1:58614" +) + +func StartMetricsServer() { + var ( + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "rushlink", + Subsystem: "shorten", + Name: "urls_total", + Help: "The current amount of shortened urls in the database.", + }, func() float64 { + var metric float64 + if err := db.DB.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte("shorten")) + if bucket == nil { + return errors.New("bucket 'shorten' could not be found") + } + metric = float64(bucket.Stats().KeyN) + return nil + }); err != nil { + log.Printf("error: %v", errors.Wrap(err, "fetching shorten_urls_total metric")) + return 0 + } + return metric + }) + ) + + router := mux.NewRouter() + router.Handle("/metrics", promhttp.Handler()).Methods("GET") + srv := &http.Server{ + Handler: router, + Addr: METRICS_ADDR, + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + log.Fatal(srv.ListenAndServe()) +} diff --git a/gobmarsh/marshal.go b/pkg/gobmarsh/marshal.go similarity index 100% rename from gobmarsh/marshal.go rename to pkg/gobmarsh/marshal.go