Add stub backends

master
Hamcha 2019-07-17 10:26:18 +02:00
parent f87c2194c8
commit c801e74db3
10 changed files with 258 additions and 22 deletions

View File

@ -1,8 +1,28 @@
# bacheca # bacheca
## What is bacheca?
RSS/Atom feed generator with a simple REST API (and hopefully gRPC) RSS/Atom feed generator with a simple REST API (and hopefully gRPC)
Written using go-kit and Moa but should work fine alone, given you have a good enough reverse proxy setup. The service is written to be able to work both as a RSS/Atom feed server and as a static generator for one.
## How does it work?
In the `cmd` folder, you'll find 3 versions of bacheca:
- `bacheca-moa` works as a microservice in a [Moa](https://git.fromouter.space/Artificiale/moa) environment
- `bacheca-server` works as a stand-alone server
- `bacheca-static` generates a static XML file
### Adding/removing items
The server version will use REST APIs for [CRUD](https://en.wikipedia.org/wiki/Create,_read,_update_and_delete) operations. [AtomPub](https://tools.ietf.org/html/rfc5023) support would be nice but has to be investigated how well it works on a server mainly doing RSS.
Another option, especially when using the static CLI, is to just make your own tools to modify the storage backend directly.
### Storage backends
Bacheca needs a backend to store its data, currently the main plan is to support at least 1 file-based backend (`file`) and a KV-based backend (`bolt`). If you want to implement and contribute your own storage backend (like PostgreSQL) feel free to!
## Things to note ## Things to note

View File

@ -8,18 +8,59 @@ import (
"os/signal" "os/signal"
"syscall" "syscall"
"git.fromouter.space/Artificiale/consulconf"
"git.fromouter.space/Artificiale/moa/sd" "git.fromouter.space/Artificiale/moa/sd"
"github.com/go-kit/kit/log" "github.com/go-kit/kit/log"
svc "git.abbiamoundominio.org/hamcha/bacheca" svc "git.abbiamoundominio.org/hamcha/bacheca"
"git.abbiamoundominio.org/hamcha/bacheca/storage"
boltbackend "git.abbiamoundominio.org/hamcha/bacheca/storage/bolt"
filebackend "git.abbiamoundominio.org/hamcha/bacheca/storage/file"
) )
type bachecaConfig struct {
Bind string
Advertise string
StorageType string
StorageFile string
}
func main() { func main() {
httpPort := flag.Int("http.port", 8080, "HTTP listen port")
consulAddr := flag.String("consul.addr", "consul:8500", "Consul address") consulAddr := flag.String("consul.addr", "consul:8500", "Consul address")
consulConf := flag.String("consul.conf", "bacheca/config", "Consul KV prefix with configuration")
flag.Parse() flag.Parse()
err := consulconf.UseDefaultClient(*consulAddr)
if err != nil {
panic(err)
}
var config bachecaConfig
err = consulconf.Get(*consulConf, &config)
if err != nil {
panic(err)
}
var backend storage.Backend
switch config.StorageType {
case "bolt":
boltdb, err := boltbackend.MakeBoltBackend(config.StorageFile)
if err != nil {
panic(err)
}
defer boltdb.Close()
backend = boltdb
case "file":
filebackend, err := filebackend.MakeFileBackend(config.StorageFile)
if err != nil {
panic(err)
}
backend = filebackend
default:
panic("unknown storage backend type (-storage.type), check help for supported backends")
}
var logger log.Logger var logger log.Logger
{ {
logger = log.NewLogfmtLogger(os.Stderr) logger = log.NewLogfmtLogger(os.Stderr)
@ -27,13 +68,13 @@ func main() {
logger = log.With(logger, "caller", log.DefaultCaller) logger = log.With(logger, "caller", log.DefaultCaller)
} }
s := svc.MakeService() s := svc.MakeService(backend)
handler := svc.MakeHTTPHandler(s, logger) handler := svc.MakeHTTPHandler(s, logger)
// Register with consul // Register with consul
registrar := sd.Register(*consulAddr, sd.Options{ registrar := sd.Register(*consulAddr, sd.Options{
Name: "bacheca", Name: "bacheca",
Advertise: fmt.Sprintf("bacheca:%d", *httpPort), Advertise: config.Advertise,
Tags: []string{ Tags: []string{
"http", // Expose via HTTP "http", // Expose via HTTP
"match-path:events", // Publish at /events/* "match-path:events", // Publish at /events/*
@ -51,9 +92,8 @@ func main() {
}() }()
go func() { go func() {
bind := fmt.Sprintf(":%d", *httpPort) logger.Log("transport", "HTTP", "addr", config.Bind)
logger.Log("transport", "HTTP", "addr", bind) errs <- http.ListenAndServe(config.Bind, handler)
errs <- http.ListenAndServe(bind, handler)
}() }()
logger.Log("exit", <-errs) logger.Log("exit", <-errs)

View File

@ -11,12 +11,36 @@ import (
"github.com/go-kit/kit/log" "github.com/go-kit/kit/log"
svc "git.abbiamoundominio.org/hamcha/bacheca" svc "git.abbiamoundominio.org/hamcha/bacheca"
"git.abbiamoundominio.org/hamcha/bacheca/storage"
boltbackend "git.abbiamoundominio.org/hamcha/bacheca/storage/bolt"
filebackend "git.abbiamoundominio.org/hamcha/bacheca/storage/file"
) )
func main() { func main() {
httpPort := flag.Int("http.port", 8080, "HTTP listen port") httpPort := flag.Int("http.port", 8080, "HTTP listen port")
dbtype := flag.String("storage.type", "bolt", "Storage backend type")
dbfile := flag.String("storage.file", "bacheca.db", "Storage file or path, for file-based backends (bolt)")
flag.Parse() flag.Parse()
var backend storage.Backend
switch *dbtype {
case "bolt":
boltdb, err := boltbackend.MakeBoltBackend(*dbfile)
if err != nil {
panic(err)
}
defer boltdb.Close()
backend = boltdb
case "file":
filebackend, err := filebackend.MakeFileBackend(*dbfile)
if err != nil {
panic(err)
}
backend = filebackend
default:
panic("unknown storage backend type (-storage.type), check help for supported backends")
}
var logger log.Logger var logger log.Logger
{ {
logger = log.NewLogfmtLogger(os.Stderr) logger = log.NewLogfmtLogger(os.Stderr)
@ -24,7 +48,7 @@ func main() {
logger = log.With(logger, "caller", log.DefaultCaller) logger = log.With(logger, "caller", log.DefaultCaller)
} }
s := svc.MakeService() s := svc.MakeService(backend)
handler := svc.MakeHTTPHandler(s, logger) handler := svc.MakeHTTPHandler(s, logger)
//TODO Add auth middleware //TODO Add auth middleware

View File

@ -4,18 +4,19 @@ import (
"context" "context"
"flag" "flag"
"fmt" "fmt"
"os"
"strings" "strings"
proto "git.abbiamoundominio.org/hamcha/bacheca/proto"
"github.com/go-kit/kit/log"
svc "git.abbiamoundominio.org/hamcha/bacheca" svc "git.abbiamoundominio.org/hamcha/bacheca"
proto "git.abbiamoundominio.org/hamcha/bacheca/proto"
"git.abbiamoundominio.org/hamcha/bacheca/storage"
boltbackend "git.abbiamoundominio.org/hamcha/bacheca/storage/bolt"
filebackend "git.abbiamoundominio.org/hamcha/bacheca/storage/file"
) )
func main() { func main() {
format := flag.String("fmt", "rss", "Feed format (rss, atom, json)") format := flag.String("fmt", "rss", "Feed format (rss, atom, json)")
dbtype := flag.String("storage.type", "bolt", "Storage backend type")
dbfile := flag.String("storage.file", "bacheca.db", "Storage file or path, for file-based backends (bolt)")
flag.Parse() flag.Parse()
ftype := strings.ToUpper(*format) ftype := strings.ToUpper(*format)
@ -24,14 +25,26 @@ func main() {
panic("invalid format (-fmt), check help for supported formats") panic("invalid format (-fmt), check help for supported formats")
} }
var logger log.Logger var backend storage.Backend
{ switch *dbtype {
logger = log.NewLogfmtLogger(os.Stderr) case "bolt":
logger = log.With(logger, "ts", log.DefaultTimestampUTC) boltdb, err := boltbackend.MakeBoltBackend(*dbfile)
logger = log.With(logger, "caller", log.DefaultCaller) if err != nil {
panic(err)
}
defer boltdb.Close()
backend = boltdb
case "file":
filebackend, err := filebackend.MakeFileBackend(*dbfile)
if err != nil {
panic(err)
}
backend = filebackend
default:
panic("unknown storage backend type (-storage.type), check help for supported backends")
} }
s := svc.MakeService() s := svc.MakeService(backend)
req := &proto.GetFeedRequest{ req := &proto.GetFeedRequest{
FeedType: proto.GetFeedRequest_FeedType(fid), FeedType: proto.GetFeedRequest_FeedType(fid),

5
go.mod
View File

@ -3,10 +3,13 @@ module git.abbiamoundominio.org/hamcha/bacheca
go 1.12 go 1.12
require ( require (
git.fromouter.space/Artificiale/consulconf v0.3.1
git.fromouter.space/Artificiale/moa v0.0.1 git.fromouter.space/Artificiale/moa v0.0.1
git.fromouter.space/mcg/cardgage v0.0.5 // indirect github.com/boltdb/bolt v1.3.1
github.com/go-kit/kit v0.8.0 github.com/go-kit/kit v0.8.0
github.com/golang/protobuf v1.3.2 github.com/golang/protobuf v1.3.2
github.com/gorilla/feeds v1.1.1
github.com/gorilla/mux v1.7.2 github.com/gorilla/mux v1.7.2
github.com/kr/pretty v0.1.0 // indirect
google.golang.org/grpc v1.22.0 google.golang.org/grpc v1.22.0
) )

8
go.sum
View File

@ -1,4 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
git.fromouter.space/Artificiale/consulconf v0.3.1 h1:WXtNqdbspFQ3KxhE5PbYt7o4YVWXInJJgT2VW3D36vo=
git.fromouter.space/Artificiale/consulconf v0.3.1/go.mod h1:hhQNBaRAwJiKe6pnwTaWcyMCGuDIRZjAa/32Hw8UY1A=
git.fromouter.space/Artificiale/moa v0.0.1-p2/go.mod h1:dHYul6vVMwDCzre18AFs6NmI22yeI7AE0iQC1jFEQi0= git.fromouter.space/Artificiale/moa v0.0.1-p2/go.mod h1:dHYul6vVMwDCzre18AFs6NmI22yeI7AE0iQC1jFEQi0=
git.fromouter.space/Artificiale/moa v0.0.1 h1:1mVML0v3Bh5Fya9dz7pKtlt7+Z6hRzXxmlY7sBvw0FI= git.fromouter.space/Artificiale/moa v0.0.1 h1:1mVML0v3Bh5Fya9dz7pKtlt7+Z6hRzXxmlY7sBvw0FI=
git.fromouter.space/Artificiale/moa v0.0.1/go.mod h1:dHYul6vVMwDCzre18AFs6NmI22yeI7AE0iQC1jFEQi0= git.fromouter.space/Artificiale/moa v0.0.1/go.mod h1:dHYul6vVMwDCzre18AFs6NmI22yeI7AE0iQC1jFEQi0=
@ -16,6 +18,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
@ -43,6 +47,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gorilla/feeds v1.1.1 h1:HwKXxqzcRNg9to+BbvJog4+f3s/xzvtZXICcQGutYfY=
github.com/gorilla/feeds v1.1.1/go.mod h1:Nk0jZrvPFZX1OBe5NPiddPw7CfwF6Q9eqzaBbaightA=
github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I=
github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@ -86,8 +92,10 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
proto "git.abbiamoundominio.org/hamcha/bacheca/proto" proto "git.abbiamoundominio.org/hamcha/bacheca/proto"
"git.abbiamoundominio.org/hamcha/bacheca/storage"
) )
// Service is the definition of the Bacheca service // Service is the definition of the Bacheca service
@ -13,10 +14,14 @@ type Service interface {
} }
type bachecaService struct { type bachecaService struct {
backend storage.Backend
} }
func MakeService() Service { // MakeService returns an instance of the bacheca service
return &bachecaService{} func MakeService(store storage.Backend) Service {
return &bachecaService{
backend: store,
}
} }
func (b *bachecaService) GetFeed(ctx context.Context, req *proto.GetFeedRequest, rsp *proto.GetFeedResponse) error { func (b *bachecaService) GetFeed(ctx context.Context, req *proto.GetFeedRequest, rsp *proto.GetFeedResponse) error {

View File

@ -0,0 +1,51 @@
package boltbackend
import (
"errors"
"github.com/boltdb/bolt"
"github.com/gorilla/feeds"
)
// BoltBackend is a Bolt-powered storage backend for bacheca
type BoltBackend struct {
db *bolt.DB
}
// MakeBoltBackend returns a BoltBackend instance
func MakeBoltBackend(path string) (*BoltBackend, error) {
db, err := bolt.Open(path, 0600, nil)
return &BoltBackend{
db: db,
}, err
}
// Close closes the underlying bolt handler
func (b *BoltBackend) Close() error {
return b.db.Close()
}
// GetFeed returns the whole feed
func (b *BoltBackend) GetFeed() (*feeds.Feed, error) {
return nil, errors.New("not implemented")
}
// AddItem adds an item to the feed
func (b *BoltBackend) AddItem(item feeds.Item) (string, error) {
return "", errors.New("not implemented")
}
// RemoveItem removes an item by ID
func (b *BoltBackend) RemoveItem(id string) error {
return errors.New("not implemented")
}
// ReplaceItem replaces the item at a specified ID with the data provided
func (b *BoltBackend) ReplaceItem(id string, item feeds.Item) error {
return errors.New("not implemented")
}
// SetInfo sets the feed metadata (like title and description)
func (b *BoltBackend) SetInfo(info feeds.Feed) error {
return errors.New("not implemented")
}

View File

@ -0,0 +1,49 @@
package filebackend
import (
"errors"
"os"
"github.com/gorilla/feeds"
)
// FileBackend is a file-based storage backend for bacheca
type FileBackend struct {
path string
}
// MakeFileBackend returns a FileBackend instance
func MakeFileBackend(path string) (*FileBackend, error) {
stat, err := os.Stat(path)
if err == nil && stat.IsDir() {
return nil, errors.New("file backend must be a file")
}
return &FileBackend{
path: path,
}, err
}
// GetFeed returns the whole feed
func (b *FileBackend) GetFeed() (*feeds.Feed, error) {
return nil, errors.New("not implemented")
}
// AddItem adds an item to the feed
func (b *FileBackend) AddItem(item feeds.Item) (string, error) {
return "", errors.New("not implemented")
}
// RemoveItem removes an item by ID
func (b *FileBackend) RemoveItem(id string) error {
return errors.New("not implemented")
}
// ReplaceItem replaces the item at a specified ID with the data provided
func (b *FileBackend) ReplaceItem(id string, item feeds.Item) error {
return errors.New("not implemented")
}
// SetInfo sets the feed metadata (like title and description)
func (b *FileBackend) SetInfo(info feeds.Feed) error {
return errors.New("not implemented")
}

23
storage/storage.go 100644
View File

@ -0,0 +1,23 @@
package storage
import "github.com/gorilla/feeds"
// Backend is a storage backend for bacheca
type Backend interface {
// GetFeed returns the whole feed
GetFeed() (*feeds.Feed, error)
// AddItem adds an item to the feed
// If an ID is not specified, one will be generated randomly
// The return value is the ID, generated or provided
AddItem(item feeds.Item) (string, error)
// RemoveItem removes an item by ID
RemoveItem(id string) error
// ReplaceItem replaces the item at a specified ID with the data provided
ReplaceItem(id string, item feeds.Item) error
// SetInfo sets the feed metadata (like title and description)
SetInfo(info feeds.Feed) error
}