From c801e74db384b565dd0c2ba0271b26291db0caa5 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Wed, 17 Jul 2019 10:26:18 +0200 Subject: [PATCH] Add stub backends --- README.md | 22 +++++++++++++++- cmd/bacheca-moa/main.go | 52 +++++++++++++++++++++++++++++++++----- cmd/bacheca-server/main.go | 26 ++++++++++++++++++- cmd/bacheca-static/main.go | 35 +++++++++++++++++-------- go.mod | 5 +++- go.sum | 8 ++++++ service.go | 9 +++++-- storage/bolt/backend.go | 51 +++++++++++++++++++++++++++++++++++++ storage/file/backend.go | 49 +++++++++++++++++++++++++++++++++++ storage/storage.go | 23 +++++++++++++++++ 10 files changed, 258 insertions(+), 22 deletions(-) create mode 100644 storage/bolt/backend.go create mode 100644 storage/file/backend.go create mode 100644 storage/storage.go diff --git a/README.md b/README.md index 5aaa929..992756a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,28 @@ # bacheca +## What is bacheca? + RSS/Atom feed generator with a simple REST API (and hopefully gRPC) -Written using go-kit and Moa but should work fine alone, given you have a good enough reverse proxy setup. +The service is written to be able to work both as a RSS/Atom feed server and as a static generator for one. + +## How does it work? + +In the `cmd` folder, you'll find 3 versions of bacheca: + + - `bacheca-moa` works as a microservice in a [Moa](https://git.fromouter.space/Artificiale/moa) environment + - `bacheca-server` works as a stand-alone server + - `bacheca-static` generates a static XML file + +### Adding/removing items + +The server version will use REST APIs for [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) operations. [AtomPub](https://tools.ietf.org/html/rfc5023) support would be nice but has to be investigated how well it works on a server mainly doing RSS. + +Another option, especially when using the static CLI, is to just make your own tools to modify the storage backend directly. + +### Storage backends + +Bacheca needs a backend to store its data, currently the main plan is to support at least 1 file-based backend (`file`) and a KV-based backend (`bolt`). If you want to implement and contribute your own storage backend (like PostgreSQL) feel free to! ## Things to note diff --git a/cmd/bacheca-moa/main.go b/cmd/bacheca-moa/main.go index 10d3d87..72b9cef 100644 --- a/cmd/bacheca-moa/main.go +++ b/cmd/bacheca-moa/main.go @@ -8,18 +8,59 @@ import ( "os/signal" "syscall" + "git.fromouter.space/Artificiale/consulconf" "git.fromouter.space/Artificiale/moa/sd" "github.com/go-kit/kit/log" svc "git.abbiamoundominio.org/hamcha/bacheca" + "git.abbiamoundominio.org/hamcha/bacheca/storage" + boltbackend "git.abbiamoundominio.org/hamcha/bacheca/storage/bolt" + filebackend "git.abbiamoundominio.org/hamcha/bacheca/storage/file" ) +type bachecaConfig struct { + Bind string + Advertise string + StorageType string + StorageFile string +} + func main() { - httpPort := flag.Int("http.port", 8080, "HTTP listen port") consulAddr := flag.String("consul.addr", "consul:8500", "Consul address") + consulConf := flag.String("consul.conf", "bacheca/config", "Consul KV prefix with configuration") flag.Parse() + err := consulconf.UseDefaultClient(*consulAddr) + if err != nil { + panic(err) + } + + var config bachecaConfig + err = consulconf.Get(*consulConf, &config) + if err != nil { + panic(err) + } + + var backend storage.Backend + switch config.StorageType { + case "bolt": + boltdb, err := boltbackend.MakeBoltBackend(config.StorageFile) + if err != nil { + panic(err) + } + defer boltdb.Close() + backend = boltdb + case "file": + filebackend, err := filebackend.MakeFileBackend(config.StorageFile) + if err != nil { + panic(err) + } + backend = filebackend + default: + panic("unknown storage backend type (-storage.type), check help for supported backends") + } + var logger log.Logger { logger = log.NewLogfmtLogger(os.Stderr) @@ -27,13 +68,13 @@ func main() { logger = log.With(logger, "caller", log.DefaultCaller) } - s := svc.MakeService() + s := svc.MakeService(backend) handler := svc.MakeHTTPHandler(s, logger) // Register with consul registrar := sd.Register(*consulAddr, sd.Options{ Name: "bacheca", - Advertise: fmt.Sprintf("bacheca:%d", *httpPort), + Advertise: config.Advertise, Tags: []string{ "http", // Expose via HTTP "match-path:events", // Publish at /events/* @@ -51,9 +92,8 @@ func main() { }() go func() { - bind := fmt.Sprintf(":%d", *httpPort) - logger.Log("transport", "HTTP", "addr", bind) - errs <- http.ListenAndServe(bind, handler) + logger.Log("transport", "HTTP", "addr", config.Bind) + errs <- http.ListenAndServe(config.Bind, handler) }() logger.Log("exit", <-errs) diff --git a/cmd/bacheca-server/main.go b/cmd/bacheca-server/main.go index 8cf34af..f914217 100644 --- a/cmd/bacheca-server/main.go +++ b/cmd/bacheca-server/main.go @@ -11,12 +11,36 @@ import ( "github.com/go-kit/kit/log" svc "git.abbiamoundominio.org/hamcha/bacheca" + "git.abbiamoundominio.org/hamcha/bacheca/storage" + boltbackend "git.abbiamoundominio.org/hamcha/bacheca/storage/bolt" + filebackend "git.abbiamoundominio.org/hamcha/bacheca/storage/file" ) func main() { httpPort := flag.Int("http.port", 8080, "HTTP listen port") + dbtype := flag.String("storage.type", "bolt", "Storage backend type") + dbfile := flag.String("storage.file", "bacheca.db", "Storage file or path, for file-based backends (bolt)") flag.Parse() + var backend storage.Backend + switch *dbtype { + case "bolt": + boltdb, err := boltbackend.MakeBoltBackend(*dbfile) + if err != nil { + panic(err) + } + defer boltdb.Close() + backend = boltdb + case "file": + filebackend, err := filebackend.MakeFileBackend(*dbfile) + if err != nil { + panic(err) + } + backend = filebackend + default: + panic("unknown storage backend type (-storage.type), check help for supported backends") + } + var logger log.Logger { logger = log.NewLogfmtLogger(os.Stderr) @@ -24,7 +48,7 @@ func main() { logger = log.With(logger, "caller", log.DefaultCaller) } - s := svc.MakeService() + s := svc.MakeService(backend) handler := svc.MakeHTTPHandler(s, logger) //TODO Add auth middleware diff --git a/cmd/bacheca-static/main.go b/cmd/bacheca-static/main.go index 17e3ff3..640b136 100644 --- a/cmd/bacheca-static/main.go +++ b/cmd/bacheca-static/main.go @@ -4,18 +4,19 @@ import ( "context" "flag" "fmt" - "os" "strings" - proto "git.abbiamoundominio.org/hamcha/bacheca/proto" - - "github.com/go-kit/kit/log" - svc "git.abbiamoundominio.org/hamcha/bacheca" + proto "git.abbiamoundominio.org/hamcha/bacheca/proto" + "git.abbiamoundominio.org/hamcha/bacheca/storage" + boltbackend "git.abbiamoundominio.org/hamcha/bacheca/storage/bolt" + filebackend "git.abbiamoundominio.org/hamcha/bacheca/storage/file" ) func main() { format := flag.String("fmt", "rss", "Feed format (rss, atom, json)") + dbtype := flag.String("storage.type", "bolt", "Storage backend type") + dbfile := flag.String("storage.file", "bacheca.db", "Storage file or path, for file-based backends (bolt)") flag.Parse() ftype := strings.ToUpper(*format) @@ -24,14 +25,26 @@ func main() { panic("invalid format (-fmt), check help for supported formats") } - var logger log.Logger - { - logger = log.NewLogfmtLogger(os.Stderr) - logger = log.With(logger, "ts", log.DefaultTimestampUTC) - logger = log.With(logger, "caller", log.DefaultCaller) + var backend storage.Backend + switch *dbtype { + case "bolt": + boltdb, err := boltbackend.MakeBoltBackend(*dbfile) + if err != nil { + panic(err) + } + defer boltdb.Close() + backend = boltdb + case "file": + filebackend, err := filebackend.MakeFileBackend(*dbfile) + if err != nil { + panic(err) + } + backend = filebackend + default: + panic("unknown storage backend type (-storage.type), check help for supported backends") } - s := svc.MakeService() + s := svc.MakeService(backend) req := &proto.GetFeedRequest{ FeedType: proto.GetFeedRequest_FeedType(fid), diff --git a/go.mod b/go.mod index fdc306b..609e979 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module git.abbiamoundominio.org/hamcha/bacheca go 1.12 require ( + git.fromouter.space/Artificiale/consulconf v0.3.1 git.fromouter.space/Artificiale/moa v0.0.1 - git.fromouter.space/mcg/cardgage v0.0.5 // indirect + github.com/boltdb/bolt v1.3.1 github.com/go-kit/kit v0.8.0 github.com/golang/protobuf v1.3.2 + github.com/gorilla/feeds v1.1.1 github.com/gorilla/mux v1.7.2 + github.com/kr/pretty v0.1.0 // indirect google.golang.org/grpc v1.22.0 ) diff --git a/go.sum b/go.sum index 0d9753f..57ad694 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +git.fromouter.space/Artificiale/consulconf v0.3.1 h1:WXtNqdbspFQ3KxhE5PbYt7o4YVWXInJJgT2VW3D36vo= +git.fromouter.space/Artificiale/consulconf v0.3.1/go.mod h1:hhQNBaRAwJiKe6pnwTaWcyMCGuDIRZjAa/32Hw8UY1A= git.fromouter.space/Artificiale/moa v0.0.1-p2/go.mod h1:dHYul6vVMwDCzre18AFs6NmI22yeI7AE0iQC1jFEQi0= git.fromouter.space/Artificiale/moa v0.0.1 h1:1mVML0v3Bh5Fya9dz7pKtlt7+Z6hRzXxmlY7sBvw0FI= git.fromouter.space/Artificiale/moa v0.0.1/go.mod h1:dHYul6vVMwDCzre18AFs6NmI22yeI7AE0iQC1jFEQi0= @@ -16,6 +18,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= @@ -43,6 +47,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY= +github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -86,8 +92,10 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= diff --git a/service.go b/service.go index a3d407f..7d8f881 100644 --- a/service.go +++ b/service.go @@ -5,6 +5,7 @@ import ( "errors" proto "git.abbiamoundominio.org/hamcha/bacheca/proto" + "git.abbiamoundominio.org/hamcha/bacheca/storage" ) // Service is the definition of the Bacheca service @@ -13,10 +14,14 @@ type Service interface { } type bachecaService struct { + backend storage.Backend } -func MakeService() Service { - return &bachecaService{} +// MakeService returns an instance of the bacheca service +func MakeService(store storage.Backend) Service { + return &bachecaService{ + backend: store, + } } func (b *bachecaService) GetFeed(ctx context.Context, req *proto.GetFeedRequest, rsp *proto.GetFeedResponse) error { diff --git a/storage/bolt/backend.go b/storage/bolt/backend.go new file mode 100644 index 0000000..8d26797 --- /dev/null +++ b/storage/bolt/backend.go @@ -0,0 +1,51 @@ +package boltbackend + +import ( + "errors" + + "github.com/boltdb/bolt" + "github.com/gorilla/feeds" +) + +// BoltBackend is a Bolt-powered storage backend for bacheca +type BoltBackend struct { + db *bolt.DB +} + +// MakeBoltBackend returns a BoltBackend instance +func MakeBoltBackend(path string) (*BoltBackend, error) { + db, err := bolt.Open(path, 0600, nil) + return &BoltBackend{ + db: db, + }, err +} + +// Close closes the underlying bolt handler +func (b *BoltBackend) Close() error { + return b.db.Close() +} + +// GetFeed returns the whole feed +func (b *BoltBackend) GetFeed() (*feeds.Feed, error) { + return nil, errors.New("not implemented") +} + +// AddItem adds an item to the feed +func (b *BoltBackend) AddItem(item feeds.Item) (string, error) { + return "", errors.New("not implemented") +} + +// RemoveItem removes an item by ID +func (b *BoltBackend) RemoveItem(id string) error { + return errors.New("not implemented") +} + +// ReplaceItem replaces the item at a specified ID with the data provided +func (b *BoltBackend) ReplaceItem(id string, item feeds.Item) error { + return errors.New("not implemented") +} + +// SetInfo sets the feed metadata (like title and description) +func (b *BoltBackend) SetInfo(info feeds.Feed) error { + return errors.New("not implemented") +} diff --git a/storage/file/backend.go b/storage/file/backend.go new file mode 100644 index 0000000..20bfc53 --- /dev/null +++ b/storage/file/backend.go @@ -0,0 +1,49 @@ +package filebackend + +import ( + "errors" + "os" + + "github.com/gorilla/feeds" +) + +// FileBackend is a file-based storage backend for bacheca +type FileBackend struct { + path string +} + +// MakeFileBackend returns a FileBackend instance +func MakeFileBackend(path string) (*FileBackend, error) { + stat, err := os.Stat(path) + if err == nil && stat.IsDir() { + return nil, errors.New("file backend must be a file") + } + return &FileBackend{ + path: path, + }, err +} + +// GetFeed returns the whole feed +func (b *FileBackend) GetFeed() (*feeds.Feed, error) { + return nil, errors.New("not implemented") +} + +// AddItem adds an item to the feed +func (b *FileBackend) AddItem(item feeds.Item) (string, error) { + return "", errors.New("not implemented") +} + +// RemoveItem removes an item by ID +func (b *FileBackend) RemoveItem(id string) error { + return errors.New("not implemented") +} + +// ReplaceItem replaces the item at a specified ID with the data provided +func (b *FileBackend) ReplaceItem(id string, item feeds.Item) error { + return errors.New("not implemented") +} + +// SetInfo sets the feed metadata (like title and description) +func (b *FileBackend) SetInfo(info feeds.Feed) error { + return errors.New("not implemented") +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..f4a3a5d --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,23 @@ +package storage + +import "github.com/gorilla/feeds" + +// Backend is a storage backend for bacheca +type Backend interface { + // GetFeed returns the whole feed + GetFeed() (*feeds.Feed, error) + + // AddItem adds an item to the feed + // If an ID is not specified, one will be generated randomly + // The return value is the ID, generated or provided + AddItem(item feeds.Item) (string, error) + + // RemoveItem removes an item by ID + RemoveItem(id string) error + + // ReplaceItem replaces the item at a specified ID with the data provided + ReplaceItem(id string, item feeds.Item) error + + // SetInfo sets the feed metadata (like title and description) + SetInfo(info feeds.Feed) error +}