diff --git a/go.mod b/go.mod index 824f2d3..85ebd77 100644 --- a/go.mod +++ b/go.mod @@ -16,11 +16,14 @@ require ( github.com/go-redis/redis/v8 v8.3.0 github.com/gomodule/redigo v1.8.2 // indirect github.com/imdario/mergo v0.3.9 + github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.4.0 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 github.com/smartystreets/goconvey v1.6.4 + github.com/syndtr/goleveldb v1.0.0 github.com/urfave/cli v1.22.3 github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 + google.golang.org/protobuf v1.23.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 ) diff --git a/go.sum b/go.sum index 4a50900..682d33c 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= @@ -114,9 +115,11 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= @@ -124,6 +127,8 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI= github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -157,6 +162,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -217,6 +224,7 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/manager/db.go b/manager/db.go index 2a8a79d..ad8fdb3 100644 --- a/manager/db.go +++ b/manager/db.go @@ -9,6 +9,8 @@ import ( "github.com/boltdb/bolt" "github.com/dgraph-io/badger/v2" "github.com/go-redis/redis/v8" + "github.com/pkg/errors" + "github.com/syndtr/goleveldb/leveldb" . "github.com/tuna/tunasync/internal" ) @@ -86,6 +88,19 @@ func makeDBAdapter(dbType string, dbFile string) (dbAdapter, error) { } err = kv.Init() return &kv, err + } else if dbType == "leveldb" { + innerDB, err := leveldb.OpenFile(dbFile, nil) + if err != nil { + return nil, err + } + db := leveldbAdapter{ + db: innerDB, + } + kv := kvDBAdapter{ + db: &db, + } + err = kv.Init() + return &kv, err } // unsupported db-type return nil, fmt.Errorf("unsupported db-type: %s", dbType) @@ -116,7 +131,7 @@ func (b *kvDBAdapter) ListWorkers() (ws []WorkerStatus, err error) { for _, v := range workers { jsonErr := json.Unmarshal(v, &w) if jsonErr != nil { - err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + err = errors.Wrap(err, jsonErr.Error()) continue } ws = append(ws, w) @@ -193,7 +208,7 @@ func (b *kvDBAdapter) ListMirrorStatus(workerID string) (ms []MirrorStatus, err var m MirrorStatus jsonErr := json.Unmarshal(v, &m) if jsonErr != nil { - err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + err = errors.Wrap(err, jsonErr.Error()) continue } ms = append(ms, m) @@ -213,7 +228,7 @@ func (b *kvDBAdapter) ListAllMirrorStatus() (ms []MirrorStatus, err error) { var m MirrorStatus jsonErr := json.Unmarshal(v, &m) if jsonErr != nil { - err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + err = errors.Wrap(err, jsonErr.Error()) continue } ms = append(ms, m) @@ -232,13 +247,13 @@ func (b *kvDBAdapter) FlushDisabledJobs() (err error) { var m MirrorStatus jsonErr := json.Unmarshal(v, &m) if jsonErr != nil { - err = fmt.Errorf("%s; %s", err.Error(), jsonErr) + err = errors.Wrap(err, jsonErr.Error()) continue } if m.Status == Disabled || len(m.Name) == 0 { deleteErr := b.db.Delete(_statusBucketKey, k) if deleteErr != nil { - err = fmt.Errorf("%s; %s", err.Error(), deleteErr) + err = errors.Wrap(err, deleteErr.Error()) } } } diff --git a/manager/db_leveldb.go b/manager/db_leveldb.go new file mode 100644 index 0000000..3cdf96f --- /dev/null +++ b/manager/db_leveldb.go @@ -0,0 +1,51 @@ +package manager + +import ( + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" +) + +// implement kv interface backed by leveldb +type leveldbAdapter struct { + db *leveldb.DB +} + +func (b *leveldbAdapter) InitBucket(bucket string) (err error) { + // no-op + return +} + +func (b *leveldbAdapter) Get(bucket string, key string) (v []byte, err error) { + v, err = b.db.Get([]byte(bucket+key), nil) + return +} + +func (b *leveldbAdapter) GetAll(bucket string) (m map[string][]byte, err error) { + it := b.db.NewIterator(util.BytesPrefix([]byte(bucket)), nil) + defer it.Release() + m = make(map[string][]byte) + for it.Next() { + k := string(it.Key()) + actualKey := k[len(bucket):] + // it.Value() changes on next iteration + val := it.Value() + v := make([]byte, len(val)) + copy(v, it.Value()) + m[actualKey] = v + } + return +} + +func (b *leveldbAdapter) Put(bucket string, key string, value []byte) error { + err := b.db.Put([]byte(bucket+key), []byte(value), nil) + return err +} + +func (b *leveldbAdapter) Delete(bucket string, key string) error { + err := b.db.Delete([]byte(bucket+key), nil) + return err +} + +func (b *leveldbAdapter) Close() error { + return b.db.Close() +} diff --git a/manager/db_test.go b/manager/db_test.go index c1ac491..1970f91 100644 --- a/manager/db_test.go +++ b/manager/db_test.go @@ -216,4 +216,22 @@ func TestDBAdapter(t *testing.T) { DBAdapterTest(badgerDB) }) + + Convey("leveldbAdapter should work", t, func() { + tmpDir, err := ioutil.TempDir("", "tunasync") + defer os.RemoveAll(tmpDir) + So(err, ShouldBeNil) + + dbType, dbFile := "leveldb", filepath.Join(tmpDir, "leveldb.db") + leveldbDB, err := makeDBAdapter(dbType, dbFile) + So(err, ShouldBeNil) + + defer func() { + // close leveldbDB + err := leveldbDB.Close() + So(err, ShouldBeNil) + }() + + DBAdapterTest(leveldbDB) + }) }