mirror of
https://github.com/tuna/tunasync.git
synced 2025-04-20 20:22:46 +00:00
Add leveldb to db backend and fix error wrapping
This commit is contained in:
Jiajie Chen
parent
932dc69ae8
commit
94154742a7
3
go.mod
3
go.mod
@ -16,11 +16,14 @@ require (
|
||||
github.com/go-redis/redis/v8 v8.3.0
|
||||
github.com/gomodule/redigo v1.8.2 // indirect
|
||||
github.com/imdario/mergo v0.3.9
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/pkg/profile v1.4.0
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46
|
||||
github.com/smartystreets/goconvey v1.6.4
|
||||
github.com/syndtr/goleveldb v1.0.0
|
||||
github.com/urfave/cli v1.22.3
|
||||
github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
|
||||
google.golang.org/protobuf v1.23.0
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
|
||||
)
|
||||
|
||||
8
go.sum
8
go.sum
@ -74,6 +74,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
|
||||
@ -114,9 +115,11 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
|
||||
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
|
||||
@ -124,6 +127,8 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI=
|
||||
github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@ -157,6 +162,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
@ -217,6 +224,7 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
@ -9,6 +9,8 @@ import (
|
||||
"github.com/boltdb/bolt"
|
||||
"github.com/dgraph-io/badger/v2"
|
||||
"github.com/go-redis/redis/v8"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
|
||||
. "github.com/tuna/tunasync/internal"
|
||||
)
|
||||
@ -86,6 +88,19 @@ func makeDBAdapter(dbType string, dbFile string) (dbAdapter, error) {
|
||||
}
|
||||
err = kv.Init()
|
||||
return &kv, err
|
||||
} else if dbType == "leveldb" {
|
||||
innerDB, err := leveldb.OpenFile(dbFile, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db := leveldbAdapter{
|
||||
db: innerDB,
|
||||
}
|
||||
kv := kvDBAdapter{
|
||||
db: &db,
|
||||
}
|
||||
err = kv.Init()
|
||||
return &kv, err
|
||||
}
|
||||
// unsupported db-type
|
||||
return nil, fmt.Errorf("unsupported db-type: %s", dbType)
|
||||
@ -116,7 +131,7 @@ func (b *kvDBAdapter) ListWorkers() (ws []WorkerStatus, err error) {
|
||||
for _, v := range workers {
|
||||
jsonErr := json.Unmarshal(v, &w)
|
||||
if jsonErr != nil {
|
||||
err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
|
||||
err = errors.Wrap(err, jsonErr.Error())
|
||||
continue
|
||||
}
|
||||
ws = append(ws, w)
|
||||
@ -193,7 +208,7 @@ func (b *kvDBAdapter) ListMirrorStatus(workerID string) (ms []MirrorStatus, err
|
||||
var m MirrorStatus
|
||||
jsonErr := json.Unmarshal(v, &m)
|
||||
if jsonErr != nil {
|
||||
err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
|
||||
err = errors.Wrap(err, jsonErr.Error())
|
||||
continue
|
||||
}
|
||||
ms = append(ms, m)
|
||||
@ -213,7 +228,7 @@ func (b *kvDBAdapter) ListAllMirrorStatus() (ms []MirrorStatus, err error) {
|
||||
var m MirrorStatus
|
||||
jsonErr := json.Unmarshal(v, &m)
|
||||
if jsonErr != nil {
|
||||
err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
|
||||
err = errors.Wrap(err, jsonErr.Error())
|
||||
continue
|
||||
}
|
||||
ms = append(ms, m)
|
||||
@ -232,13 +247,13 @@ func (b *kvDBAdapter) FlushDisabledJobs() (err error) {
|
||||
var m MirrorStatus
|
||||
jsonErr := json.Unmarshal(v, &m)
|
||||
if jsonErr != nil {
|
||||
err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
|
||||
err = errors.Wrap(err, jsonErr.Error())
|
||||
continue
|
||||
}
|
||||
if m.Status == Disabled || len(m.Name) == 0 {
|
||||
deleteErr := b.db.Delete(_statusBucketKey, k)
|
||||
if deleteErr != nil {
|
||||
err = fmt.Errorf("%s; %s", err.Error(), deleteErr)
|
||||
err = errors.Wrap(err, deleteErr.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
51
manager/db_leveldb.go
Normal file
51
manager/db_leveldb.go
Normal file
@ -0,0 +1,51 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
"github.com/syndtr/goleveldb/leveldb/util"
|
||||
)
|
||||
|
||||
// implement kv interface backed by leveldb
|
||||
type leveldbAdapter struct {
|
||||
db *leveldb.DB
|
||||
}
|
||||
|
||||
func (b *leveldbAdapter) InitBucket(bucket string) (err error) {
|
||||
// no-op
|
||||
return
|
||||
}
|
||||
|
||||
func (b *leveldbAdapter) Get(bucket string, key string) (v []byte, err error) {
|
||||
v, err = b.db.Get([]byte(bucket+key), nil)
|
||||
return
|
||||
}
|
||||
|
||||
func (b *leveldbAdapter) GetAll(bucket string) (m map[string][]byte, err error) {
|
||||
it := b.db.NewIterator(util.BytesPrefix([]byte(bucket)), nil)
|
||||
defer it.Release()
|
||||
m = make(map[string][]byte)
|
||||
for it.Next() {
|
||||
k := string(it.Key())
|
||||
actualKey := k[len(bucket):]
|
||||
// it.Value() changes on next iteration
|
||||
val := it.Value()
|
||||
v := make([]byte, len(val))
|
||||
copy(v, it.Value())
|
||||
m[actualKey] = v
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (b *leveldbAdapter) Put(bucket string, key string, value []byte) error {
|
||||
err := b.db.Put([]byte(bucket+key), []byte(value), nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *leveldbAdapter) Delete(bucket string, key string) error {
|
||||
err := b.db.Delete([]byte(bucket+key), nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *leveldbAdapter) Close() error {
|
||||
return b.db.Close()
|
||||
}
|
||||
@ -216,4 +216,22 @@ func TestDBAdapter(t *testing.T) {
|
||||
|
||||
DBAdapterTest(badgerDB)
|
||||
})
|
||||
|
||||
Convey("leveldbAdapter should work", t, func() {
|
||||
tmpDir, err := ioutil.TempDir("", "tunasync")
|
||||
defer os.RemoveAll(tmpDir)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
dbType, dbFile := "leveldb", filepath.Join(tmpDir, "leveldb.db")
|
||||
leveldbDB, err := makeDBAdapter(dbType, dbFile)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
defer func() {
|
||||
// close leveldbDB
|
||||
err := leveldbDB.Close()
|
||||
So(err, ShouldBeNil)
|
||||
}()
|
||||
|
||||
DBAdapterTest(leveldbDB)
|
||||
})
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user