pax_global_header00006660000000000000000000000064141114767710014523gustar00rootroot0000000000000052 comment=378be8c6a90250ae209df59b218c2b1bad32d89d nosql-0.3.8/000077500000000000000000000000001411147677100126675ustar00rootroot00000000000000nosql-0.3.8/.github/000077500000000000000000000000001411147677100142275ustar00rootroot00000000000000nosql-0.3.8/.github/ISSUE_TEMPLATE/000077500000000000000000000000001411147677100164125ustar00rootroot00000000000000nosql-0.3.8/.github/ISSUE_TEMPLATE/bug-report.md000066400000000000000000000010521411147677100210200ustar00rootroot00000000000000--- name: Bug Report about: Create a report to help us improve title: '' labels: bug, needs triage assignees: '' --- ### Subject of the issue Describe your issue here. ### Your environment * OS - * Version - ### Steps to reproduce Tell us how to reproduce this issue. Please provide a working demo, you can use [this template](https://plnkr.co/edit/XorWgI?p=preview) as a base. ### Expected behaviour Tell us what should happen ### Actual behaviour Tell us what happens instead ### Additional context Add any other context about the problem here. nosql-0.3.8/.github/ISSUE_TEMPLATE/documentation-request.md000066400000000000000000000006451411147677100233000ustar00rootroot00000000000000--- name: Documentation Request about: Request documentation for a feature title: '' labels: needs triage assignees: '' --- nosql-0.3.8/.github/ISSUE_TEMPLATE/enhancement.md000066400000000000000000000002721411147677100212220ustar00rootroot00000000000000--- name: Enhancement about: Suggest an enhancement to nosql title: '' labels: enhancement, needs triage assignees: '' --- ### What would you like to be added ### Why this is needed nosql-0.3.8/.github/labeler.yml000066400000000000000000000001761411147677100163640ustar00rootroot00000000000000needs triage: - '**' # index.php | src/main.php - '.*' # .gitignore - '.*/**' # .github/workflows/label.yml nosql-0.3.8/.github/workflows/000077500000000000000000000000001411147677100162645ustar00rootroot00000000000000nosql-0.3.8/.github/workflows/labeler.yml000066400000000000000000000003141411147677100204130ustar00rootroot00000000000000name: Pull Request Labeler on: pull_request_target jobs: label: runs-on: ubuntu-latest steps: - uses: actions/labeler@v3.0.2 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" nosql-0.3.8/.gitignore000066400000000000000000000003341411147677100146570ustar00rootroot00000000000000# Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Vendor directory vendor/*nosql-0.3.8/.golangci.yml000066400000000000000000000034311411147677100152540ustar00rootroot00000000000000linters-settings: govet: check-shadowing: true settings: printf: funcs: - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf golint: min-confidence: 0 gocyclo: min-complexity: 10 maligned: suggest-new: true dupl: threshold: 100 goconst: min-len: 2 min-occurrences: 2 depguard: list-type: blacklist packages: # logging is allowed only by logutils.Log, logrus # is allowed to use only in logutils package - github.com/sirupsen/logrus misspell: locale: US lll: line-length: 140 goimports: local-prefixes: github.com/golangci/golangci-lint gocritic: enabled-tags: - performance - style - experimental disabled-checks: - wrapperFunc - dupImport # https://github.com/go-critic/go-critic/issues/845 linters: disable-all: true enable: - gofmt - golint - vet - misspell - ineffassign - deadcode - staticcheck - unused - structcheck run: skip-dirs: - pkg issues: exclude: - can't lint - declaration of "err" shadows declaration at line - should have a package comment, unless it's in another file for this package - error strings should not be capitalized or end with punctuation or a newline # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration service: golangci-lint-version: 1.18.x # use the fixed version to not introduce new linters unexpectedly prepare: - echo "here I can run custom commands, but no preparation needed for this repo" nosql-0.3.8/.travis.yml000066400000000000000000000005331411147677100150010ustar00rootroot00000000000000language: go go: - 1.14.x services: - mysql addons: apt: packages: - debhelper - fakeroot - bash-completion env: global: - V=1 before_script: - make bootstrap script: - TRAVIS=1 make after_success: - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" notifications: email: false nosql-0.3.8/LICENSE000066400000000000000000000261321411147677100137000ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright (c) 2019 Smallstep Labs, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. nosql-0.3.8/Makefile000066400000000000000000000034501411147677100143310ustar00rootroot00000000000000# Set V to 1 for verbose output from the Makefile Q=$(if $V,,@) PREFIX?= SRC=$(shell find . -type f -name '*.go' -not -path "./vendor/*") GOOS_OVERRIDE ?= OUTPUT_ROOT=output/ all: test lint travis: travis-test lint .PHONY: all ######################################### # Bootstrapping ######################################### bootstra%: $Q GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.24.0 .PHONY: bootstra% ################################################# # Determine the type of `push` and `version` ################################################# # Version flags to embed in the binaries VERSION ?= $(shell [ -d .git ] && git describe --tags --always --dirty="-dev") # If we are not in an active git dir then try reading the version from .VERSION. # .VERSION contains a slug populated by `git archive`. VERSION := $(or $(VERSION),$(shell ./.version.sh .VERSION)) VERSION := $(shell echo $(VERSION) | sed 's/^v//') NOT_RC := $(shell echo $(VERSION) | grep -v -e -rc) # If TRAVIS_TAG is set then we know this ref has been tagged. ifdef TRAVIS_TAG ifeq ($(NOT_RC),) PUSHTYPE=release-candidate else PUSHTYPE=release endif else PUSHTYPE=master endif ######################################### # Test ######################################### test: $Q $(GOFLAGS) go test -short -coverprofile=coverage.out ./... travis-test: $Q $(GOFLAGS) TRAVIS=1 go test -short -coverprofile=coverage.out ./... .PHONY: test travis-test ######################################### # Linting ######################################### fmt: $Q gofmt -l -w $(SRC) lint: $Q LOG_LEVEL=error golangci-lint run .PHONY: lint fmt ######################################### # Clean ######################################### clean: ifneq ($(BINNAME),"") $Q rm -f bin/$(BINNAME) endif .PHONY: clean nosql-0.3.8/README.md000066400000000000000000000006411411147677100141470ustar00rootroot00000000000000# NoSQL NoSQL is an abstraction layer for data persistence. This project is in development, the API is not stable. # Implementations The current version comes with a few implementations inlcuding Mysql, Badger, and BoltDB, but implementations are on the roadmap. - [ ] Memory - [x] [BoltDB](https://github.com/etcd-io/bbolt) etcd fork. - [x] Badger - [x] MariaDB/MySQL - [ ] PostgreSQL - [ ] Cassandra - [ ] ... nosql-0.3.8/badger/000077500000000000000000000000001411147677100141135ustar00rootroot00000000000000nosql-0.3.8/badger/v1/000077500000000000000000000000001411147677100144415ustar00rootroot00000000000000nosql-0.3.8/badger/v1/badger.go000066400000000000000000000256561411147677100162320ustar00rootroot00000000000000package badger import ( "bytes" "encoding/binary" "strings" "github.com/dgraph-io/badger" "github.com/dgraph-io/badger/options" "github.com/pkg/errors" "github.com/smallstep/nosql/database" ) // DB is a wrapper over *badger.DB, type DB struct { db *badger.DB } // Open opens or creates a BoltDB database in the given path. func (db *DB) Open(dir string, opt ...database.Option) (err error) { opts := &database.Options{} for _, o := range opt { if err := o(opts); err != nil { return err } } bo := badger.DefaultOptions(dir) // Set the Table and Value LoadingMode - default is MemoryMap. Low memory/RAM // systems may want to use FileIO. switch strings.ToLower(opts.BadgerFileLoadingMode) { case "", database.BadgerMemoryMap, "memorymap": bo.TableLoadingMode = options.MemoryMap bo.ValueLogLoadingMode = options.MemoryMap case database.BadgerFileIO: bo.TableLoadingMode = options.FileIO bo.ValueLogLoadingMode = options.FileIO default: return badger.ErrInvalidLoadingMode } if opts.ValueDir != "" { bo.ValueDir = opts.ValueDir } else { bo.ValueDir = dir } db.db, err = badger.Open(bo) return errors.Wrap(err, "error opening Badger database") } // Close closes the DB database. func (db *DB) Close() error { return errors.Wrap(db.db.Close(), "error closing Badger database") } // CreateTable creates a token element with the 'bucket' prefix so that such // that their appears to be a table. func (db *DB) CreateTable(bucket []byte) error { bk, err := badgerEncode(bucket) if err != nil { return err } return db.db.Update(func(txn *badger.Txn) error { return errors.Wrapf(txn.Set(bk, []byte{}), "failed to create %s/", bucket) }) } // DeleteTable deletes a root or embedded bucket. Returns an error if the // bucket cannot be found or if the key represents a non-bucket value. func (db *DB) DeleteTable(bucket []byte) error { var tableExists bool prefix, err := badgerEncode(bucket) if err != nil { return err } deleteKeys := func(keysForDelete [][]byte) error { if err := db.db.Update(func(txn *badger.Txn) error { for _, key := range keysForDelete { tableExists = true if err := txn.Delete(key); err != nil { return errors.Wrapf(err, "error deleting key %s", key) } } return nil }); err != nil { return errors.Wrapf(err, "update failed") } return nil } collectSize := 1000 err = db.db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions opts.AllVersions = false opts.PrefetchValues = false it := txn.NewIterator(opts) defer it.Close() keysForDelete := make([][]byte, collectSize) keysCollected := 0 for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { key := it.Item().KeyCopy(nil) keysForDelete[keysCollected] = key keysCollected++ if keysCollected == collectSize { if err := deleteKeys(keysForDelete); err != nil { return err } keysCollected = 0 } } if keysCollected > 0 { if err := deleteKeys(keysForDelete[:keysCollected]); err != nil { return err } } if !tableExists { return errors.Wrapf(database.ErrNotFound, "table %s does not exist", bucket) } return nil }) return err } // badgerGet is a helper for the Get method. func badgerGet(txn *badger.Txn, key []byte) ([]byte, error) { item, err := txn.Get(key) switch { case err == badger.ErrKeyNotFound: return nil, errors.Wrapf(database.ErrNotFound, "key %s not found", key) case err != nil: return nil, errors.Wrapf(err, "failed to get key %s", key) default: val, err := item.ValueCopy(nil) if err != nil { return nil, errors.Wrap(err, "error accessing value returned by database") } return val, nil } } // Get returns the value stored in the given bucked and key. func (db *DB) Get(bucket, key []byte) (ret []byte, err error) { bk, err := toBadgerKey(bucket, key) if err != nil { return nil, errors.Wrapf(err, "error converting %s/%s to badgerKey", bucket, key) } err = db.db.View(func(txn *badger.Txn) error { ret, err = badgerGet(txn, bk) return err }) return } // Set stores the given value on bucket and key. func (db *DB) Set(bucket, key, value []byte) error { bk, err := toBadgerKey(bucket, key) if err != nil { return errors.Wrapf(err, "error converting %s/%s to badgerKey", bucket, key) } return db.db.Update(func(txn *badger.Txn) error { return errors.Wrapf(txn.Set(bk, value), "failed to set %s/%s", bucket, key) }) } // Del deletes the value stored in the given bucked and key. func (db *DB) Del(bucket, key []byte) error { bk, err := toBadgerKey(bucket, key) if err != nil { return errors.Wrapf(err, "error converting %s/%s to badgerKey", bucket, key) } return db.db.Update(func(txn *badger.Txn) error { return errors.Wrapf(txn.Delete(bk), "failed to delete %s/%s", bucket, key) }) } // List returns the full list of entries in a bucket. func (db *DB) List(bucket []byte) ([]*database.Entry, error) { var ( entries []*database.Entry tableExists bool ) err := db.db.View(func(txn *badger.Txn) error { it := txn.NewIterator(badger.DefaultIteratorOptions) defer it.Close() prefix, err := badgerEncode(bucket) if err != nil { return err } for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { tableExists = true item := it.Item() bk := item.KeyCopy(nil) if isBadgerTable(bk) { continue } _bucket, key, err := fromBadgerKey(bk) if err != nil { return errors.Wrapf(err, "error converting from badgerKey %s", bk) } if !bytes.Equal(_bucket, bucket) { return errors.Errorf("bucket names do not match; want %v, but got %v", bucket, _bucket) } v, err := item.ValueCopy(nil) if err != nil { return errors.Wrap(err, "error retrieving contents from database value") } entries = append(entries, &database.Entry{ Bucket: _bucket, Key: key, Value: v, }) } if !tableExists { return errors.Wrapf(database.ErrNotFound, "bucket %s not found", bucket) } return nil }) return entries, err } // CmpAndSwap modifies the value at the given bucket and key (to newValue) // only if the existing (current) value matches oldValue. func (db *DB) CmpAndSwap(bucket, key, oldValue, newValue []byte) ([]byte, bool, error) { bk, err := toBadgerKey(bucket, key) if err != nil { return nil, false, err } badgerTxn := db.db.NewTransaction(true) defer badgerTxn.Discard() val, swapped, err := cmpAndSwap(badgerTxn, bk, oldValue, newValue) switch { case err != nil: return nil, false, err case swapped: if err := badgerTxn.Commit(); err != nil { return nil, false, errors.Wrapf(err, "failed to commit badger transaction") } return val, swapped, nil default: return val, swapped, err } } func cmpAndSwap(badgerTxn *badger.Txn, bk, oldValue, newValue []byte) ([]byte, bool, error) { current, err := badgerGet(badgerTxn, bk) // If value does not exist but expected is not nil, then return w/out swapping. if err != nil && !database.IsErrNotFound(err) { return nil, false, err } if !bytes.Equal(current, oldValue) { return current, false, nil } if err := badgerTxn.Set(bk, newValue); err != nil { return current, false, errors.Wrapf(err, "failed to set %s", bk) } return newValue, true, nil } // Update performs multiple commands on one read-write transaction. func (db *DB) Update(txn *database.Tx) error { return db.db.Update(func(badgerTxn *badger.Txn) (err error) { for _, q := range txn.Operations { switch q.Cmd { case database.CreateTable: if err = db.CreateTable(q.Bucket); err != nil { return err } continue case database.DeleteTable: if err = db.DeleteTable(q.Bucket); err != nil { return err } continue } bk, err := toBadgerKey(q.Bucket, q.Key) if err != nil { return err } switch q.Cmd { case database.Get: if q.Result, err = badgerGet(badgerTxn, bk); err != nil { return errors.Wrapf(err, "failed to get %s/%s", q.Bucket, q.Key) } case database.Set: if err := badgerTxn.Set(bk, q.Value); err != nil { return errors.Wrapf(err, "failed to set %s/%s", q.Bucket, q.Key) } case database.Delete: if err = badgerTxn.Delete(bk); err != nil { return errors.Wrapf(err, "failed to delete %s/%s", q.Bucket, q.Key) } case database.CmpAndSwap: q.Result, q.Swapped, err = cmpAndSwap(badgerTxn, bk, q.CmpValue, q.Value) if err != nil { return errors.Wrapf(err, "failed to CmpAndSwap %s/%s", q.Bucket, q.Key) } case database.CmpOrRollback: return database.ErrOpNotSupported default: return database.ErrOpNotSupported } } return nil }) } // toBadgerKey returns the Badger database key using the following algorithm: // First 2 bytes are the length of the bucket/table name in little endian format, // followed by the bucket/table name, // followed by 2 bytes representing the length of the key in little endian format, // followed by the key. func toBadgerKey(bucket, key []byte) ([]byte, error) { first, err := badgerEncode(bucket) if err != nil { return nil, err } second, err := badgerEncode(key) if err != nil { return nil, err } return append(first, second...), nil } // isBadgerTable returns True if the slice is a badgerTable token, false otherwise. // badgerTable means that the slice contains only the [size|value] of one section // of a badgerKey and no remainder. A badgerKey is [buket|key], while a badgerTable // is only the bucket section. func isBadgerTable(bk []byte) bool { if k, rest := parseBadgerEncode(bk); len(k) > 0 && len(rest) == 0 { return true } return false } // fromBadgerKey returns the bucket and key encoded in a BadgerKey. // See documentation for toBadgerKey. func fromBadgerKey(bk []byte) ([]byte, []byte, error) { bucket, rest := parseBadgerEncode(bk) if len(bucket) == 0 || len(rest) == 0 { return nil, nil, errors.Errorf("invalid badger key: %v", bk) } key, rest2 := parseBadgerEncode(rest) if len(key) == 0 || len(rest2) != 0 { return nil, nil, errors.Errorf("invalid badger key: %v", bk) } return bucket, key, nil } // badgerEncode encodes a byte slice into a section of a BadgerKey. // See documentation for toBadgerKey. func badgerEncode(val []byte) ([]byte, error) { l := len(val) switch { case l == 0: return nil, errors.Errorf("input cannot be empty") case l > 65535: return nil, errors.Errorf("length of input cannot be greater than 65535") default: lb := new(bytes.Buffer) if err := binary.Write(lb, binary.LittleEndian, uint16(l)); err != nil { return nil, errors.Wrap(err, "error doing binary Write") } return append(lb.Bytes(), val...), nil } } func parseBadgerEncode(bk []byte) (value, rest []byte) { var ( keyLen uint16 start = uint16(2) length = uint16(len(bk)) ) if uint16(len(bk)) < start { return nil, bk } // First 2 bytes stores the length of the value. if err := binary.Read(bytes.NewReader(bk[:2]), binary.LittleEndian, &keyLen); err != nil { return nil, bk } end := start + keyLen switch { case length < end: return nil, bk case length == end: return bk[start:end], nil default: return bk[start:end], bk[end:] } } nosql-0.3.8/badger/v1/badger_test.go000066400000000000000000000116351411147677100172610ustar00rootroot00000000000000package badger import ( "errors" "testing" "github.com/smallstep/assert" ) func Test_badgerEncode(t *testing.T) { type args struct { val []byte } tests := []struct { name string args args want []byte err error }{ { name: "fail/input-too-long", args: args{make([]byte, 65536)}, err: errors.New("length of input cannot be greater than 65535"), }, { name: "fail/input-empty", args: args{nil}, err: errors.New("input cannot be empty"), }, { name: "ok", args: args{[]byte("hello")}, want: []byte{5, 0, 104, 101, 108, 108, 111}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := badgerEncode(tt.args.val) if err != nil { if assert.NotNil(t, tt.err) { assert.HasPrefix(t, err.Error(), tt.err.Error()) } } else { if assert.Nil(t, tt.err) && assert.NotNil(t, got) && assert.NotNil(t, tt.want) { assert.Equals(t, got, tt.want) } } }) } } func Test_toBadgerKey(t *testing.T) { type args struct { bucket []byte key []byte } tests := []struct { name string args args want []byte err error }{ { name: "fail/bucket-too-long", args: args{make([]byte, 65536), []byte("goodbye")}, err: errors.New("length of input cannot be greater than 65535"), }, { name: "fail/key-empty", args: args{[]byte("hello"), nil}, err: errors.New("input cannot be empty"), }, { name: "ok", args: args{[]byte("hello"), []byte("goodbye")}, want: []byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := toBadgerKey(tt.args.bucket, tt.args.key) if err != nil { if assert.NotNil(t, tt.err) { assert.HasPrefix(t, err.Error(), tt.err.Error()) } } else { if assert.Nil(t, tt.err) && assert.NotNil(t, got) && assert.NotNil(t, tt.want) { assert.Equals(t, got, tt.want) } } }) } } func Test_fromBadgerKey(t *testing.T) { type args struct { bk []byte } type ret struct { bucket []byte key []byte } tests := []struct { name string args args want ret err error }{ { name: "fail/input-too-short/no-bucket-length", args: args{[]byte{5}}, err: errors.New("invalid badger key: [5]"), }, { name: "fail/input-too-short/no-key-length", args: args{[]byte{5, 0, 104, 101, 108, 108, 111}}, err: errors.New("invalid badger key: [5 0 104 101 108 108 111]"), }, { name: "fail/input-too-short/invalid-key", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103}}, err: errors.New("invalid badger key: [5 0 104 101 108 108 111 7 0 103]"), }, { name: "ok", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}}, want: ret{[]byte{104, 101, 108, 108, 111}, []byte{103, 111, 111, 100, 98, 121, 101}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bucket, key, err := fromBadgerKey(tt.args.bk) if err != nil { if assert.NotNil(t, tt.err) { assert.HasPrefix(t, err.Error(), tt.err.Error()) } } else { if assert.Nil(t, tt.err) && assert.NotNil(t, bucket) && assert.NotNil(t, key) { assert.Equals(t, bucket, tt.want.bucket) assert.Equals(t, key, tt.want.key) } } }) } } func Test_parseBadgerEncode(t *testing.T) { type args struct { bk []byte } type ret struct { bucket []byte key []byte } tests := []struct { name string args args want ret }{ { name: "fail/keylen-too-short", args: args{[]byte{5}}, want: ret{nil, []byte{5}}, }, { name: "fail/key-too-short", args: args{[]byte{5, 0, 111, 111}}, want: ret{nil, []byte{5, 0, 111, 111}}, }, { name: "ok/exact-length", args: args{[]byte{5, 0, 104, 101, 108, 108, 111}}, want: ret{[]byte{104, 101, 108, 108, 111}, nil}, }, { name: "ok/longer", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}}, want: ret{[]byte{104, 101, 108, 108, 111}, []byte{7, 0, 103, 111, 111, 100, 98, 121, 101}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bucket, key := parseBadgerEncode(tt.args.bk) assert.Equals(t, bucket, tt.want.bucket) assert.Equals(t, key, tt.want.key) }) } } func Test_isBadgerTable(t *testing.T) { type args struct { bk []byte } tests := []struct { name string args args want bool }{ { name: "false/keylen-too-short", args: args{[]byte{5}}, want: false, }, { name: "false/key-too-short", args: args{[]byte{5, 0, 111, 111}}, want: false, }, { name: "ok", args: args{[]byte{5, 0, 104, 101, 108, 108, 111}}, want: true, }, { name: "false/key-too-long", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}}, want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equals(t, isBadgerTable(tt.args.bk), tt.want) }) } } nosql-0.3.8/badger/v2/000077500000000000000000000000001411147677100144425ustar00rootroot00000000000000nosql-0.3.8/badger/v2/badger.go000066400000000000000000000261171411147677100162240ustar00rootroot00000000000000package badger import ( "bytes" "encoding/binary" "strings" "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/options" "github.com/pkg/errors" "github.com/smallstep/nosql/database" ) // DB is a wrapper over *badger/v2.DB, type DB struct { db *badger.DB } // Open opens or creates a BoltDB database in the given path. func (db *DB) Open(dir string, opt ...database.Option) (err error) { opts := &database.Options{} for _, o := range opt { if err := o(opts); err != nil { return err } } bo := badger.DefaultOptions(dir) if opts.ValueDir != "" { bo.ValueDir = opts.ValueDir } // Set the ValueLogLoadingMode - default is MemoryMap. Low memory/RAM // systems may want to use FileIO. switch strings.ToLower(opts.BadgerFileLoadingMode) { case "", database.BadgerMemoryMap, "memorymap": bo.ValueLogLoadingMode = options.MemoryMap case database.BadgerFileIO: bo.ValueLogLoadingMode = options.FileIO default: return badger.ErrInvalidLoadingMode } db.db, err = badger.Open(bo) return errors.Wrap(err, "error opening Badger database") } // Close closes the DB database. func (db *DB) Close() error { return errors.Wrap(db.db.Close(), "error closing Badger database") } // CreateTable creates a token element with the 'bucket' prefix so that such // that their appears to be a table. func (db *DB) CreateTable(bucket []byte) error { bk, err := badgerEncode(bucket) if err != nil { return err } return db.db.Update(func(txn *badger.Txn) error { return errors.Wrapf(txn.Set(bk, []byte{}), "failed to create %s/", bucket) }) } // DeleteTable deletes a root or embedded bucket. Returns an error if the // bucket cannot be found or if the key represents a non-bucket value. func (db *DB) DeleteTable(bucket []byte) error { var tableExists bool prefix, err := badgerEncode(bucket) if err != nil { return err } deleteKeys := func(keysForDelete [][]byte) error { if err := db.db.Update(func(txn *badger.Txn) error { for _, key := range keysForDelete { tableExists = true if err := txn.Delete(key); err != nil { return errors.Wrapf(err, "error deleting key %s", key) } } return nil }); err != nil { return errors.Wrapf(err, "update failed") } return nil } collectSize := 1000 err = db.db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions opts.AllVersions = false opts.PrefetchValues = false it := txn.NewIterator(opts) defer it.Close() keysForDelete := make([][]byte, collectSize) keysCollected := 0 for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { key := it.Item().KeyCopy(nil) keysForDelete[keysCollected] = key keysCollected++ if keysCollected == collectSize { if err := deleteKeys(keysForDelete); err != nil { return err } keysCollected = 0 } } if keysCollected > 0 { if err := deleteKeys(keysForDelete[:keysCollected]); err != nil { return err } } if !tableExists { return errors.Wrapf(database.ErrNotFound, "table %s does not exist", bucket) } return nil }) return err } // badgerGetV2 is a helper for the Get method. func badgerGetV2(txn *badger.Txn, key []byte) ([]byte, error) { item, err := txn.Get(key) switch { case err == badger.ErrKeyNotFound: return nil, errors.Wrapf(database.ErrNotFound, "key %s not found", key) case err != nil: return nil, errors.Wrapf(err, "failed to get key %s", key) default: val, err := item.ValueCopy(nil) if err != nil { return nil, errors.Wrap(err, "error accessing value returned by database") } // Make sure to return a copy as val is only valid during the // transaction. return cloneBytes(val), nil } } // Get returns the value stored in the given bucked and key. func (db *DB) Get(bucket, key []byte) (ret []byte, err error) { bk, err := toBadgerKey(bucket, key) if err != nil { return nil, errors.Wrapf(err, "error converting %s/%s to badgerKey", bucket, key) } err = db.db.View(func(txn *badger.Txn) error { ret, err = badgerGetV2(txn, bk) return err }) return } // Set stores the given value on bucket and key. func (db *DB) Set(bucket, key, value []byte) error { bk, err := toBadgerKey(bucket, key) if err != nil { return errors.Wrapf(err, "error converting %s/%s to badgerKey", bucket, key) } return db.db.Update(func(txn *badger.Txn) error { return errors.Wrapf(txn.Set(bk, value), "failed to set %s/%s", bucket, key) }) } // Del deletes the value stored in the given bucked and key. func (db *DB) Del(bucket, key []byte) error { bk, err := toBadgerKey(bucket, key) if err != nil { return errors.Wrapf(err, "error converting %s/%s to badgerKey", bucket, key) } return db.db.Update(func(txn *badger.Txn) error { return errors.Wrapf(txn.Delete(bk), "failed to delete %s/%s", bucket, key) }) } // List returns the full list of entries in a bucket. func (db *DB) List(bucket []byte) ([]*database.Entry, error) { var ( entries []*database.Entry tableExists bool ) err := db.db.View(func(txn *badger.Txn) error { it := txn.NewIterator(badger.DefaultIteratorOptions) defer it.Close() prefix, err := badgerEncode(bucket) if err != nil { return err } for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { tableExists = true item := it.Item() bk := item.KeyCopy(nil) if isBadgerTable(bk) { continue } _bucket, key, err := fromBadgerKey(bk) if err != nil { return errors.Wrapf(err, "error converting from badgerKey %s", bk) } if !bytes.Equal(_bucket, bucket) { return errors.Errorf("bucket names do not match; want %v, but got %v", bucket, _bucket) } v, err := item.ValueCopy(nil) if err != nil { return errors.Wrap(err, "error retrieving contents from database value") } entries = append(entries, &database.Entry{ Bucket: _bucket, Key: key, Value: cloneBytes(v), }) } if !tableExists { return errors.Wrapf(database.ErrNotFound, "bucket %s not found", bucket) } return nil }) return entries, err } // CmpAndSwap modifies the value at the given bucket and key (to newValue) // only if the existing (current) value matches oldValue. func (db *DB) CmpAndSwap(bucket, key, oldValue, newValue []byte) ([]byte, bool, error) { bk, err := toBadgerKey(bucket, key) if err != nil { return nil, false, err } badgerTxn := db.db.NewTransaction(true) defer badgerTxn.Discard() val, swapped, err := cmpAndSwapV2(badgerTxn, bk, oldValue, newValue) switch { case err != nil: return nil, false, err case swapped: if err := badgerTxn.Commit(); err != nil { return nil, false, errors.Wrapf(err, "failed to commit badger transaction") } return val, swapped, nil default: return val, swapped, err } } func cmpAndSwapV2(badgerTxn *badger.Txn, bk, oldValue, newValue []byte) ([]byte, bool, error) { current, err := badgerGetV2(badgerTxn, bk) // If value does not exist but expected is not nil, then return w/out swapping. if err != nil && !database.IsErrNotFound(err) { return nil, false, err } if !bytes.Equal(current, oldValue) { return current, false, nil } if err := badgerTxn.Set(bk, newValue); err != nil { return current, false, errors.Wrapf(err, "failed to set %s", bk) } return newValue, true, nil } // Update performs multiple commands on one read-write transaction. func (db *DB) Update(txn *database.Tx) error { return db.db.Update(func(badgerTxn *badger.Txn) (err error) { for _, q := range txn.Operations { switch q.Cmd { case database.CreateTable: if err = db.CreateTable(q.Bucket); err != nil { return err } continue case database.DeleteTable: if err = db.DeleteTable(q.Bucket); err != nil { return err } continue } bk, err := toBadgerKey(q.Bucket, q.Key) if err != nil { return err } switch q.Cmd { case database.Get: if q.Result, err = badgerGetV2(badgerTxn, bk); err != nil { return errors.Wrapf(err, "failed to get %s/%s", q.Bucket, q.Key) } case database.Set: if err := badgerTxn.Set(bk, q.Value); err != nil { return errors.Wrapf(err, "failed to set %s/%s", q.Bucket, q.Key) } case database.Delete: if err = badgerTxn.Delete(bk); err != nil { return errors.Wrapf(err, "failed to delete %s/%s", q.Bucket, q.Key) } case database.CmpAndSwap: q.Result, q.Swapped, err = cmpAndSwapV2(badgerTxn, bk, q.CmpValue, q.Value) if err != nil { return errors.Wrapf(err, "failed to CmpAndSwap %s/%s", q.Bucket, q.Key) } case database.CmpOrRollback: return database.ErrOpNotSupported default: return database.ErrOpNotSupported } } return nil }) } // toBadgerKey returns the Badger database key using the following algorithm: // First 2 bytes are the length of the bucket/table name in little endian format, // followed by the bucket/table name, // followed by 2 bytes representing the length of the key in little endian format, // followed by the key. func toBadgerKey(bucket, key []byte) ([]byte, error) { first, err := badgerEncode(bucket) if err != nil { return nil, err } second, err := badgerEncode(key) if err != nil { return nil, err } return append(first, second...), nil } // isBadgerTable returns True if the slice is a badgerTable token, false otherwise. // badgerTable means that the slice contains only the [size|value] of one section // of a badgerKey and no remainder. A badgerKey is [buket|key], while a badgerTable // is only the bucket section. func isBadgerTable(bk []byte) bool { if k, rest := parseBadgerEncode(bk); len(k) > 0 && len(rest) == 0 { return true } return false } // fromBadgerKey returns the bucket and key encoded in a BadgerKey. // See documentation for toBadgerKey. func fromBadgerKey(bk []byte) ([]byte, []byte, error) { bucket, rest := parseBadgerEncode(bk) if len(bucket) == 0 || len(rest) == 0 { return nil, nil, errors.Errorf("invalid badger key: %v", bk) } key, rest2 := parseBadgerEncode(rest) if len(key) == 0 || len(rest2) != 0 { return nil, nil, errors.Errorf("invalid badger key: %v", bk) } return bucket, key, nil } // badgerEncode encodes a byte slice into a section of a BadgerKey. // See documentation for toBadgerKey. func badgerEncode(val []byte) ([]byte, error) { l := len(val) switch { case l == 0: return nil, errors.Errorf("input cannot be empty") case l > 65535: return nil, errors.Errorf("length of input cannot be greater than 65535") default: lb := new(bytes.Buffer) if err := binary.Write(lb, binary.LittleEndian, uint16(l)); err != nil { return nil, errors.Wrap(err, "error doing binary Write") } return append(lb.Bytes(), val...), nil } } func parseBadgerEncode(bk []byte) (value, rest []byte) { var ( keyLen uint16 start = uint16(2) length = uint16(len(bk)) ) if uint16(len(bk)) < start { return nil, bk } // First 2 bytes stores the length of the value. if err := binary.Read(bytes.NewReader(bk[:2]), binary.LittleEndian, &keyLen); err != nil { return nil, bk } end := start + keyLen switch { case length < end: return nil, bk case length == end: return bk[start:end], nil default: return bk[start:end], bk[end:] } } // cloneBytes returns a copy of a given slice. func cloneBytes(v []byte) []byte { var clone = make([]byte, len(v)) copy(clone, v) return clone } nosql-0.3.8/badger/v2/badger_test.go000066400000000000000000000116351411147677100172620ustar00rootroot00000000000000package badger import ( "errors" "testing" "github.com/smallstep/assert" ) func Test_badgerEncode(t *testing.T) { type args struct { val []byte } tests := []struct { name string args args want []byte err error }{ { name: "fail/input-too-long", args: args{make([]byte, 65536)}, err: errors.New("length of input cannot be greater than 65535"), }, { name: "fail/input-empty", args: args{nil}, err: errors.New("input cannot be empty"), }, { name: "ok", args: args{[]byte("hello")}, want: []byte{5, 0, 104, 101, 108, 108, 111}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := badgerEncode(tt.args.val) if err != nil { if assert.NotNil(t, tt.err) { assert.HasPrefix(t, err.Error(), tt.err.Error()) } } else { if assert.Nil(t, tt.err) && assert.NotNil(t, got) && assert.NotNil(t, tt.want) { assert.Equals(t, got, tt.want) } } }) } } func Test_toBadgerKey(t *testing.T) { type args struct { bucket []byte key []byte } tests := []struct { name string args args want []byte err error }{ { name: "fail/bucket-too-long", args: args{make([]byte, 65536), []byte("goodbye")}, err: errors.New("length of input cannot be greater than 65535"), }, { name: "fail/key-empty", args: args{[]byte("hello"), nil}, err: errors.New("input cannot be empty"), }, { name: "ok", args: args{[]byte("hello"), []byte("goodbye")}, want: []byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := toBadgerKey(tt.args.bucket, tt.args.key) if err != nil { if assert.NotNil(t, tt.err) { assert.HasPrefix(t, err.Error(), tt.err.Error()) } } else { if assert.Nil(t, tt.err) && assert.NotNil(t, got) && assert.NotNil(t, tt.want) { assert.Equals(t, got, tt.want) } } }) } } func Test_fromBadgerKey(t *testing.T) { type args struct { bk []byte } type ret struct { bucket []byte key []byte } tests := []struct { name string args args want ret err error }{ { name: "fail/input-too-short/no-bucket-length", args: args{[]byte{5}}, err: errors.New("invalid badger key: [5]"), }, { name: "fail/input-too-short/no-key-length", args: args{[]byte{5, 0, 104, 101, 108, 108, 111}}, err: errors.New("invalid badger key: [5 0 104 101 108 108 111]"), }, { name: "fail/input-too-short/invalid-key", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103}}, err: errors.New("invalid badger key: [5 0 104 101 108 108 111 7 0 103]"), }, { name: "ok", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}}, want: ret{[]byte{104, 101, 108, 108, 111}, []byte{103, 111, 111, 100, 98, 121, 101}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bucket, key, err := fromBadgerKey(tt.args.bk) if err != nil { if assert.NotNil(t, tt.err) { assert.HasPrefix(t, err.Error(), tt.err.Error()) } } else { if assert.Nil(t, tt.err) && assert.NotNil(t, bucket) && assert.NotNil(t, key) { assert.Equals(t, bucket, tt.want.bucket) assert.Equals(t, key, tt.want.key) } } }) } } func Test_parseBadgerEncode(t *testing.T) { type args struct { bk []byte } type ret struct { bucket []byte key []byte } tests := []struct { name string args args want ret }{ { name: "fail/keylen-too-short", args: args{[]byte{5}}, want: ret{nil, []byte{5}}, }, { name: "fail/key-too-short", args: args{[]byte{5, 0, 111, 111}}, want: ret{nil, []byte{5, 0, 111, 111}}, }, { name: "ok/exact-length", args: args{[]byte{5, 0, 104, 101, 108, 108, 111}}, want: ret{[]byte{104, 101, 108, 108, 111}, nil}, }, { name: "ok/longer", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}}, want: ret{[]byte{104, 101, 108, 108, 111}, []byte{7, 0, 103, 111, 111, 100, 98, 121, 101}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { bucket, key := parseBadgerEncode(tt.args.bk) assert.Equals(t, bucket, tt.want.bucket) assert.Equals(t, key, tt.want.key) }) } } func Test_isBadgerTable(t *testing.T) { type args struct { bk []byte } tests := []struct { name string args args want bool }{ { name: "false/keylen-too-short", args: args{[]byte{5}}, want: false, }, { name: "false/key-too-short", args: args{[]byte{5, 0, 111, 111}}, want: false, }, { name: "ok", args: args{[]byte{5, 0, 104, 101, 108, 108, 111}}, want: true, }, { name: "false/key-too-long", args: args{[]byte{5, 0, 104, 101, 108, 108, 111, 7, 0, 103, 111, 111, 100, 98, 121, 101}}, want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equals(t, isBadgerTable(tt.args.bk), tt.want) }) } } nosql-0.3.8/bolt/000077500000000000000000000000001411147677100136275ustar00rootroot00000000000000nosql-0.3.8/bolt/bbolt.go000066400000000000000000000164151411147677100152670ustar00rootroot00000000000000package bolt import ( "bytes" "time" "github.com/pkg/errors" "github.com/smallstep/nosql/database" bolt "go.etcd.io/bbolt" ) var boltDBSep = []byte("/") // DB is a wrapper over bolt.DB, type DB struct { db *bolt.DB } type boltBucket interface { Bucket(name []byte) *bolt.Bucket CreateBucket(name []byte) (*bolt.Bucket, error) CreateBucketIfNotExists(name []byte) (*bolt.Bucket, error) DeleteBucket(name []byte) error } // Open opens or creates a DB database in the given path. func (db *DB) Open(dataSourceName string, opt ...database.Option) (err error) { opts := &database.Options{} for _, o := range opt { if err := o(opts); err != nil { return err } } db.db, err = bolt.Open(dataSourceName, 0600, &bolt.Options{Timeout: 5 * time.Second}) return errors.WithStack(err) } // Close closes the DB database. func (db *DB) Close() error { return errors.WithStack(db.db.Close()) } // CreateTable creates a bucket or an embedded bucket if it does not exists. func (db *DB) CreateTable(bucket []byte) error { return db.db.Update(func(tx *bolt.Tx) error { return db.createBucket(tx, bucket) }) } // DeleteTable deletes a root or embedded bucket. Returns an error if the // bucket cannot be found or if the key represents a non-bucket value. func (db *DB) DeleteTable(bucket []byte) error { return db.db.Update(func(tx *bolt.Tx) error { return db.deleteBucket(tx, bucket) }) } // Get returns the value stored in the given bucked and key. func (db *DB) Get(bucket, key []byte) (ret []byte, err error) { err = db.db.View(func(tx *bolt.Tx) error { b, err := db.getBucket(tx, bucket) if err != nil { return err } ret = b.Get(key) if ret == nil { return database.ErrNotFound } // Make sure to return a copy as ret is only valid during the // transaction. ret = cloneBytes(ret) return nil }) return } // Set stores the given value on bucket and key. func (db *DB) Set(bucket, key, value []byte) error { return db.db.Update(func(tx *bolt.Tx) error { b, err := db.getBucket(tx, bucket) if err != nil { return err } return errors.WithStack(b.Put(key, value)) }) } // Del deletes the value stored in the given bucked and key. func (db *DB) Del(bucket, key []byte) error { return db.db.Update(func(tx *bolt.Tx) error { b, err := db.getBucket(tx, bucket) if err != nil { return err } return errors.WithStack(b.Delete(key)) }) } // List returns the full list of entries in a bucket. func (db *DB) List(bucket []byte) ([]*database.Entry, error) { var entries []*database.Entry err := db.db.View(func(tx *bolt.Tx) error { b, err := db.getBucket(tx, bucket) if err != nil { return errors.Wrap(err, "getBucket failed") } c := b.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { entries = append(entries, &database.Entry{ Bucket: bucket, Key: cloneBytes(k), Value: cloneBytes(v), }) } return nil }) return entries, err } // CmpAndSwap modifies the value at the given bucket and key (to newValue) // only if the existing (current) value matches oldValue. func (db *DB) CmpAndSwap(bucket, key, oldValue, newValue []byte) ([]byte, bool, error) { boltTx, err := db.db.Begin(true) if err != nil { return nil, false, errors.Wrap(err, "error creating Bolt transaction") } boltBucket := boltTx.Bucket(bucket) if boltBucket == nil { return nil, false, errors.Errorf("failed to get bucket %s", bucket) } val, swapped, err := cmpAndSwap(boltBucket, key, oldValue, newValue) switch { case err != nil: if err := boltTx.Rollback(); err != nil { return nil, false, errors.Wrapf(err, "failed to execute CmpAndSwap transaction on %s/%s and failed to rollback transaction", bucket, key) } return nil, false, err case swapped: if err := boltTx.Commit(); err != nil { return nil, false, errors.Wrapf(err, "failed to commit badger transaction") } return val, swapped, nil default: if err := boltTx.Rollback(); err != nil { return nil, false, errors.Wrapf(err, "failed to rollback read-only CmpAndSwap transaction on %s/%s", bucket, key) } return val, swapped, err } } func cmpAndSwap(boltBucket *bolt.Bucket, key, oldValue, newValue []byte) ([]byte, bool, error) { current := boltBucket.Get(key) if !bytes.Equal(current, oldValue) { return cloneBytes(current), false, nil } if err := boltBucket.Put(key, newValue); err != nil { return nil, false, errors.Wrapf(err, "failed to set key %s", key) } return newValue, true, nil } // Update performs multiple commands on one read-write transaction. func (db *DB) Update(tx *database.Tx) error { return db.db.Update(func(boltTx *bolt.Tx) (err error) { var b *bolt.Bucket for _, q := range tx.Operations { // create or delete buckets switch q.Cmd { case database.CreateTable: err = db.createBucket(boltTx, q.Bucket) if err != nil { return err } continue case database.DeleteTable: err = db.deleteBucket(boltTx, q.Bucket) if err != nil { return err } continue } // For other operations, get bucket and perform operation b = boltTx.Bucket(q.Bucket) switch q.Cmd { case database.Get: ret := b.Get(q.Key) if ret == nil { return errors.WithStack(database.ErrNotFound) } q.Result = cloneBytes(ret) case database.Set: if err = b.Put(q.Key, q.Value); err != nil { return errors.WithStack(err) } case database.Delete: if err = b.Delete(q.Key); err != nil { return errors.WithStack(err) } case database.CmpAndSwap: q.Result, q.Swapped, err = cmpAndSwap(b, q.Key, q.CmpValue, q.Value) if err != nil { return errors.Wrapf(err, "failed to execute CmpAndSwap on %s/%s", q.Bucket, q.Key) } case database.CmpOrRollback: return errors.Errorf("operation '%s' is not yet implemented", q.Cmd) default: return errors.Errorf("operation '%s' is not supported", q.Cmd) } } return nil }) } // getBucket returns the bucket supporting nested buckets, nested buckets are // bucket names separated by '/'. func (db *DB) getBucket(tx *bolt.Tx, name []byte) (b *bolt.Bucket, err error) { buckets := bytes.Split(name, boltDBSep) for i, n := range buckets { if i == 0 { b = tx.Bucket(n) } else { b = b.Bucket(n) } if b == nil { return nil, database.ErrNotFound } } return } // createBucket creates a bucket or a nested bucket in the given transaction. func (db *DB) createBucket(tx *bolt.Tx, name []byte) (err error) { b := boltBucket(tx) buckets := bytes.Split(name, boltDBSep) for _, name := range buckets { b, err = b.CreateBucketIfNotExists(name) if err != nil { return errors.WithStack(err) } } return } // deleteBucket deletes a bucket or a nested bucked in the given transaction. func (db *DB) deleteBucket(tx *bolt.Tx, name []byte) (err error) { b := boltBucket(tx) buckets := bytes.Split(name, boltDBSep) last := len(buckets) - 1 for i := 0; i < last; i++ { if buck := b.Bucket(buckets[i]); buck == nil { return errors.Wrapf(database.ErrNotFound, "bucket %s does not exist", bytes.Join(buckets[0:i+1], boltDBSep)) } } err = b.DeleteBucket(buckets[last]) if err == bolt.ErrBucketNotFound { return errors.Wrapf(database.ErrNotFound, "bucket %s does not exist", name) } return } // cloneBytes returns a copy of a given slice. func cloneBytes(v []byte) []byte { var clone = make([]byte, len(v)) copy(clone, v) return clone } nosql-0.3.8/database/000077500000000000000000000000001411147677100144335ustar00rootroot00000000000000nosql-0.3.8/database/database.go000066400000000000000000000155131411147677100165330ustar00rootroot00000000000000package database import ( "fmt" "errors" ) var ( // ErrNotFound is the type returned on DB implementations if an item does not // exist. ErrNotFound = errors.New("not found") // ErrOpNotSupported is the type returned on DB implementations if an operation // is not supported. ErrOpNotSupported = errors.New("operation not supported") ) // IsErrNotFound returns true if the cause of the given error is ErrNotFound. func IsErrNotFound(err error) bool { return err == ErrNotFound || cause(err) == ErrNotFound } // IsErrOpNotSupported returns true if the cause of the given error is ErrOpNotSupported. func IsErrOpNotSupported(err error) bool { return err == ErrOpNotSupported || cause(err) == ErrNotFound } // cause (from github.com/pkg/errors) returns the underlying cause of the // error, if possible. An error value has a cause if it implements the // following interface: // // type causer interface { // Cause() error // } // // If the error does not implement Cause, the original error will // be returned. If the error is nil, nil will be returned without further // investigation. func cause(err error) error { type causer interface { Cause() error } for err != nil { cause, ok := err.(causer) if !ok { break } err = cause.Cause() } return err } // Options are configuration options for the database. type Options struct { Database string ValueDir string BadgerFileLoadingMode string } // Option is the modifier type over Options. type Option func(o *Options) error // WithValueDir is a modifier that sets the ValueDir attribute of Options. func WithValueDir(path string) Option { return func(o *Options) error { o.ValueDir = path return nil } } // WithDatabase is a modifier that sets the Database attribute of Options. func WithDatabase(db string) Option { return func(o *Options) error { o.Database = db return nil } } // WithBadgerFileLoadingMode is a modifier that sets the ValueLogLoadingMode // of Badger db. func WithBadgerFileLoadingMode(mode string) Option { return func(o *Options) error { o.BadgerFileLoadingMode = mode return nil } } // DB is a interface to be implemented by the databases. type DB interface { // Open opens the database available with the given options. Open(dataSourceName string, opt ...Option) error // Close closes the current database. Close() error // Get returns the value stored in the given table/bucket and key. Get(bucket, key []byte) (ret []byte, err error) // Set sets the given value in the given table/bucket and key. Set(bucket, key, value []byte) error // CmpAndSwap swaps the value at the given bucket and key if the current // value is equivalent to the oldValue input. Returns 'true' if the // swap was successful and 'false' otherwise. CmpAndSwap(bucket, key, oldValue, newValue []byte) ([]byte, bool, error) // Del deletes the data in the given table/bucket and key. Del(bucket, key []byte) error // List returns a list of all the entries in a given table/bucket. List(bucket []byte) ([]*Entry, error) // Update performs a transaction with multiple read-write commands. Update(tx *Tx) error // CreateTable creates a table or a bucket in the database. CreateTable(bucket []byte) error // DeleteTable deletes a table or a bucket in the database. DeleteTable(bucket []byte) error } // Badger FileLoadingMode constants. const ( BadgerMemoryMap = "mmap" BadgerFileIO = "fileio" ) // TxCmd is the type used to represent database command and operations. type TxCmd int const ( // CreateTable on a TxEntry will represent the creation of a table or // bucket on the database. CreateTable TxCmd = iota // DeleteTable on a TxEntry will represent the deletion of a table or // bucket on the database. DeleteTable // Get on a TxEntry will represent a command to retrieve data from the // database. Get // Set on a TxEntry will represent a command to write data on the // database. Set // Delete on a TxEntry represent a command to delete data on the database. Delete // CmpAndSwap on a TxEntry will represent a compare and swap operation on // the database. It will compare the value read and change it if it's // different. The TxEntry will contain the value read. CmpAndSwap // CmpOrRollback on a TxEntry will represent a read transaction that will // compare the values will the ones passed, and if they don't match the // transaction will fail CmpOrRollback ) // String implements the fmt.Stringer interface on TxCmd. func (o TxCmd) String() string { switch o { case CreateTable: return "create-table" case DeleteTable: return "delete-table" case Get: return "read" case Set: return "write" case Delete: return "delete" case CmpAndSwap: return "compare-and-swap" case CmpOrRollback: return "compare-and-rollback" default: return fmt.Sprintf("unknown(%d)", o) } } // Tx represents a transaction and it's list of multiple TxEntry. Each TxEntry // represents a read or write operation on the database. type Tx struct { Operations []*TxEntry } // CreateTable adds a new create query to the transaction. func (tx *Tx) CreateTable(bucket []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Cmd: CreateTable, }) } // DeleteTable adds a new create query to the transaction. func (tx *Tx) DeleteTable(bucket []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Cmd: DeleteTable, }) } // Get adds a new read query to the transaction. func (tx *Tx) Get(bucket, key []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Key: key, Cmd: Get, }) } // Set adds a new write query to the transaction. func (tx *Tx) Set(bucket, key, value []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Key: key, Value: value, Cmd: Set, }) } // Del adds a new delete query to the transaction. func (tx *Tx) Del(bucket, key []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Key: key, Cmd: Delete, }) } // Cas adds a new compare-and-swap query to the transaction. func (tx *Tx) Cas(bucket, key, value []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Key: key, Value: value, Cmd: CmpAndSwap, }) } // Cmp adds a new compare-or-rollback query to the transaction. func (tx *Tx) Cmp(bucket, key, value []byte) { tx.Operations = append(tx.Operations, &TxEntry{ Bucket: bucket, Key: key, Value: value, Cmd: CmpOrRollback, }) } // TxEntry is the base elements for the transactions, a TxEntry is a read or // write operation on the database. type TxEntry struct { Bucket []byte Key []byte Value []byte CmpValue []byte // Where the result of Get or CmpAndSwap txns is stored. Result []byte Cmd TxCmd Swapped bool } // Entry is the return value for list commands. type Entry struct { Bucket []byte Key []byte Value []byte } nosql-0.3.8/go.mod000066400000000000000000000011201411147677100137670ustar00rootroot00000000000000module github.com/smallstep/nosql go 1.14 require ( github.com/dgraph-io/badger v1.6.2 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/golang/protobuf v1.4.3 // indirect github.com/pkg/errors v0.9.1 github.com/smallstep/assert v0.0.0-20180720014142-de77670473b5 go.etcd.io/bbolt v1.3.5 golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect google.golang.org/protobuf v1.25.0 // indirect ) nosql-0.3.8/go.sum000066400000000000000000000355641411147677100140370ustar00rootroot00000000000000cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/smallstep/assert v0.0.0-20180720014142-de77670473b5 h1:lX6ybsQW9Agn3qK/W1Z39Z4a6RyEMGem/gXUYW0axYk= github.com/smallstep/assert v0.0.0-20180720014142-de77670473b5/go.mod h1:TC9A4+RjIOS+HyTH7wG17/gSqVv95uDw2J64dQZx7RE= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= nosql-0.3.8/mysql/000077500000000000000000000000001411147677100140345ustar00rootroot00000000000000nosql-0.3.8/mysql/mysql.go000066400000000000000000000173171411147677100155410ustar00rootroot00000000000000package mysql import ( "bytes" "database/sql" "fmt" "strings" // import mysql driver anonymously (just run the init) _ "github.com/go-sql-driver/mysql" "github.com/pkg/errors" "github.com/smallstep/nosql/database" ) // DB is a wrapper over *sql.DB, type DB struct { db *sql.DB } // Open creates a Driver and connects to the database with the given address // and access details. func (db *DB) Open(dataSourceName string, opt ...database.Option) error { opts := &database.Options{} for _, o := range opt { if err := o(opts); err != nil { return err } } var err error _db, err := sql.Open("mysql", dataSourceName) if err != nil { return errors.Wrap(err, "error connecting to mysql") } _, err = _db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", opts.Database)) if err != nil { return errors.Wrapf(err, "error creating database %s (if not exists)", opts.Database) } db.db, err = sql.Open("mysql", dataSourceName+opts.Database) if err != nil { return errors.Wrapf(err, "error connecting to mysql database") } return nil } // Close shutsdown the database driver. func (db *DB) Close() error { return errors.WithStack(db.db.Close()) } func getQry(bucket []byte) string { return fmt.Sprintf("SELECT nvalue FROM `%s` WHERE nkey = ?", bucket) } func insertUpdateQry(bucket []byte) string { return fmt.Sprintf("INSERT INTO `%s`(nkey, nvalue) VALUES(?,?) ON DUPLICATE KEY UPDATE nvalue = ?", bucket) } func delQry(bucket []byte) string { return fmt.Sprintf("DELETE FROM `%s` WHERE nkey = ?", bucket) } func createTableQry(bucket []byte) string { return fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s`(nkey VARBINARY(255), nvalue BLOB, PRIMARY KEY (nkey));", bucket) } func deleteTableQry(bucket []byte) string { return fmt.Sprintf("DROP TABLE `%s`", bucket) } // Get retrieves the column/row with given key. func (db *DB) Get(bucket, key []byte) ([]byte, error) { var val string err := db.db.QueryRow(getQry(bucket), key).Scan(&val) switch { case err == sql.ErrNoRows: return nil, errors.Wrapf(database.ErrNotFound, "%s/%s not found", bucket, key) case err != nil: return nil, errors.Wrapf(err, "failed to get %s/%s", bucket, key) default: return []byte(val), nil } } // Set inserts the key and value into the given bucket(column). func (db *DB) Set(bucket, key, value []byte) error { _, err := db.db.Exec(insertUpdateQry(bucket), key, value, value) if err != nil { return errors.Wrapf(err, "failed to set %s/%s", bucket, key) } return nil } // Del deletes a row from the database. func (db *DB) Del(bucket, key []byte) error { _, err := db.db.Exec(delQry(bucket), key) return errors.Wrapf(err, "failed to delete %s/%s", bucket, key) } // List returns the full list of entries in a column. func (db *DB) List(bucket []byte) ([]*database.Entry, error) { rows, err := db.db.Query(fmt.Sprintf("SELECT * FROM `%s`", bucket)) if err != nil { estr := err.Error() if strings.HasPrefix(estr, "Error 1146:") { return nil, errors.Wrapf(database.ErrNotFound, estr) } return nil, errors.Wrapf(err, "error querying table %s", bucket) } defer rows.Close() var ( key, value string entries []*database.Entry ) for rows.Next() { err := rows.Scan(&key, &value) if err != nil { return nil, errors.Wrap(err, "error getting key and value from row") } entries = append(entries, &database.Entry{ Bucket: bucket, Key: []byte(key), Value: []byte(value), }) } err = rows.Err() if err != nil { return nil, errors.Wrap(err, "error accessing row") } return entries, nil } // CmpAndSwap modifies the value at the given bucket and key (to newValue) // only if the existing (current) value matches oldValue. func (db *DB) CmpAndSwap(bucket, key, oldValue, newValue []byte) ([]byte, bool, error) { sqlTx, err := db.db.Begin() if err != nil { return nil, false, errors.WithStack(err) } val, swapped, err := cmpAndSwap(sqlTx, bucket, key, oldValue, newValue) switch { case err != nil: if err := sqlTx.Rollback(); err != nil { return nil, false, errors.Wrapf(err, "failed to execute CmpAndSwap transaction on %s/%s and failed to rollback transaction", bucket, key) } return nil, false, err case swapped: if err := sqlTx.Commit(); err != nil { return nil, false, errors.Wrapf(err, "failed to commit badger transaction") } return val, swapped, nil default: if err := sqlTx.Rollback(); err != nil { return nil, false, errors.Wrapf(err, "failed to rollback read-only CmpAndSwap transaction on %s/%s", bucket, key) } return val, swapped, err } } func cmpAndSwap(sqlTx *sql.Tx, bucket, key, oldValue, newValue []byte) ([]byte, bool, error) { var current []byte err := sqlTx.QueryRow(getQry(bucket), key).Scan(¤t) if err != nil && err != sql.ErrNoRows { return nil, false, err } if !bytes.Equal(current, oldValue) { return current, false, nil } if _, err = sqlTx.Exec(insertUpdateQry(bucket), key, newValue, newValue); err != nil { return nil, false, errors.Wrapf(err, "failed to set %s/%s", bucket, key) } return newValue, true, nil } // Update performs multiple commands on one read-write transaction. func (db *DB) Update(tx *database.Tx) error { sqlTx, err := db.db.Begin() if err != nil { return errors.WithStack(err) } rollback := func(err error) error { if rollbackErr := sqlTx.Rollback(); rollbackErr != nil { return errors.Wrap(err, "UPDATE failed, unable to rollback transaction") } return errors.Wrap(err, "UPDATE failed") } for _, q := range tx.Operations { // create or delete buckets switch q.Cmd { case database.CreateTable: _, err := sqlTx.Exec(createTableQry(q.Bucket)) if err != nil { return rollback(errors.Wrapf(err, "failed to create table %s", q.Bucket)) } case database.DeleteTable: _, err := sqlTx.Exec(deleteTableQry(q.Bucket)) if err != nil { estr := err.Error() if strings.HasPrefix(err.Error(), "Error 1051:") { return errors.Wrapf(database.ErrNotFound, estr) } return errors.Wrapf(err, "failed to delete table %s", q.Bucket) } case database.Get: var val string err := sqlTx.QueryRow(getQry(q.Bucket), q.Key).Scan(&val) switch { case err == sql.ErrNoRows: return rollback(errors.Wrapf(database.ErrNotFound, "%s/%s not found", q.Bucket, q.Key)) case err != nil: return rollback(errors.Wrapf(err, "failed to get %s/%s", q.Bucket, q.Key)) default: q.Result = []byte(val) } case database.Set: if _, err = sqlTx.Exec(insertUpdateQry(q.Bucket), q.Key, q.Value, q.Value); err != nil { return rollback(errors.Wrapf(err, "failed to set %s/%s", q.Bucket, q.Key)) } case database.Delete: if _, err = sqlTx.Exec(delQry(q.Bucket), q.Key); err != nil { return rollback(errors.Wrapf(err, "failed to delete %s/%s", q.Bucket, q.Key)) } case database.CmpAndSwap: q.Result, q.Swapped, err = cmpAndSwap(sqlTx, q.Bucket, q.Key, q.CmpValue, q.Value) if err != nil { return rollback(errors.Wrapf(err, "failed to load-or-store %s/%s", q.Bucket, q.Key)) } case database.CmpOrRollback: return database.ErrOpNotSupported default: return database.ErrOpNotSupported } } if err = errors.WithStack(sqlTx.Commit()); err != nil { return rollback(err) } return nil } // CreateTable creates a table in the database. func (db *DB) CreateTable(bucket []byte) error { _, err := db.db.Exec(createTableQry(bucket)) if err != nil { return errors.Wrapf(err, "failed to create table %s", bucket) } return nil } // DeleteTable deletes a table in the database. func (db *DB) DeleteTable(bucket []byte) error { _, err := db.db.Exec(deleteTableQry(bucket)) if err != nil { estr := err.Error() if strings.HasPrefix(err.Error(), "Error 1051:") { return errors.Wrapf(database.ErrNotFound, estr) } return errors.Wrapf(err, "failed to delete table %s", bucket) } return nil } nosql-0.3.8/nosql.go000066400000000000000000000041771411147677100143630ustar00rootroot00000000000000package nosql import ( "strings" "github.com/pkg/errors" badgerV1 "github.com/smallstep/nosql/badger/v1" badgerV2 "github.com/smallstep/nosql/badger/v2" "github.com/smallstep/nosql/bolt" "github.com/smallstep/nosql/database" "github.com/smallstep/nosql/mysql" ) // Option is just a wrapper over database.Option. type Option = database.Option // DB is just a wrapper over database.DB. type DB = database.DB var ( // WithValueDir is a wrapper over database.WithValueDir. WithValueDir = database.WithValueDir // WithDatabase is a wrapper over database.WithDatabase. WithDatabase = database.WithDatabase // WithBadgerFileLoadingMode is a wrapper over database.WithBadgerFileLoadingMode. WithBadgerFileLoadingMode = database.WithBadgerFileLoadingMode // IsErrNotFound is a wrapper over database.IsErrNotFound. IsErrNotFound = database.IsErrNotFound // IsErrOpNotSupported is a wrapper over database.IsErrOpNotSupported. IsErrOpNotSupported = database.IsErrOpNotSupported // Available db driver types. // // BadgerDriver indicates the default Badger database - currently Badger V1. BadgerDriver = "badger" // BadgerV1Driver explicitly selects the Badger V1 driver. BadgerV1Driver = "badgerv1" // BadgerV2Driver explicitly selects the Badger V2 driver. BadgerV2Driver = "badgerv2" // BBoltDriver indicates the default BBolt database. BBoltDriver = "bbolt" // MySQLDriver indicates the default MySQL database. MySQLDriver = "mysql" // Badger FileLoadingMode // BadgerMemoryMap indicates the MemoryMap FileLoadingMode option. BadgerMemoryMap = database.BadgerMemoryMap // BadgerFileIO indicates the FileIO FileLoadingMode option. BadgerFileIO = database.BadgerFileIO ) // New returns a database with the given driver. func New(driver, dataSourceName string, opt ...Option) (db database.DB, err error) { switch strings.ToLower(driver) { case BadgerDriver, BadgerV1Driver: db = &badgerV1.DB{} case BadgerV2Driver: db = &badgerV2.DB{} case BBoltDriver: db = &bolt.DB{} case MySQLDriver: db = &mysql.DB{} default: return nil, errors.Errorf("%s database not supported", driver) } err = db.Open(dataSourceName, opt...) return } nosql-0.3.8/nosql_test.go000066400000000000000000000174741411147677100154260ustar00rootroot00000000000000package nosql import ( "encoding/json" "fmt" "os" "testing" "github.com/smallstep/assert" "github.com/smallstep/nosql/database" ) type testUser struct { Fname, lname string numPets int } func run(t *testing.T, db database.DB) { var boogers = []byte("boogers") ub := []byte("testNoSQLUsers") assert.True(t, IsErrNotFound(db.DeleteTable(ub))) assert.Nil(t, db.CreateTable(ub)) // Verify that re-creating the table does not cause a "table already exists" error assert.Nil(t, db.CreateTable(ub)) // Test that we can create tables with illegal/special characters (e.g. `-`) illName := []byte("test-special-char") assert.Nil(t, db.CreateTable(illName)) assert.Nil(t, db.DeleteTable(illName)) _, err := db.List(illName) assert.True(t, IsErrNotFound(err)) // List should be empty entries, err := db.List(ub) assert.Nil(t, err) assert.Equals(t, len(entries), 0) // check for mike - should not exist _, err = db.Get(ub, []byte("mike")) assert.True(t, IsErrNotFound(err)) // add mike assert.Nil(t, db.Set(ub, []byte("mike"), boogers)) // verify that mike is in db res, err := db.Get(ub, []byte("mike")) assert.FatalError(t, err) assert.Equals(t, boogers, res) // overwrite mike mike := testUser{"mike", "malone", 1} mikeb, err := json.Marshal(mike) assert.FatalError(t, err) assert.Nil(t, db.Set(ub, []byte("mike"), mikeb)) // verify overwrite res, err = db.Get(ub, []byte("mike")) assert.FatalError(t, err) assert.Equals(t, mikeb, res) var swapped bool // CmpAndSwap should load since mike is not nil res, swapped, err = db.CmpAndSwap(ub, []byte("mike"), nil, boogers) assert.FatalError(t, err) assert.Equals(t, mikeb, res) assert.False(t, swapped) assert.Nil(t, err) // delete mike assert.FatalError(t, db.Del(ub, []byte("mike"))) // CmpAndSwap should overwrite mike since mike is nil res, swapped, err = db.CmpAndSwap(ub, []byte("mike"), nil, boogers) assert.FatalError(t, err) assert.Equals(t, boogers, res) assert.True(t, swapped) assert.Nil(t, err) // delete mike assert.FatalError(t, db.Del(ub, []byte("mike"))) // check for mike - should not exist _, err = db.Get(ub, []byte("mike")) assert.True(t, IsErrNotFound(err)) // CmpAndSwap should store since mike does not exist res, swapped, err = db.CmpAndSwap(ub, []byte("mike"), nil, mikeb) assert.FatalError(t, err) assert.Equals(t, res, mikeb) assert.True(t, swapped) assert.Nil(t, err) // delete mike assert.FatalError(t, db.Del(ub, []byte("mike"))) // Update // // create txns for update test mariano := testUser{"mariano", "Cano", 2} marianob, err := json.Marshal(mariano) assert.FatalError(t, err) seb := testUser{"sebastian", "tiedtke", 0} sebb, err := json.Marshal(seb) assert.FatalError(t, err) gates := testUser{"bill", "gates", 2} gatesb, err := json.Marshal(gates) assert.FatalError(t, err) casGates := &database.TxEntry{ Bucket: ub, Key: []byte("bill"), Value: gatesb, CmpValue: nil, Cmd: database.CmpAndSwap, } setMike := &database.TxEntry{ Bucket: ub, Key: []byte("mike"), Value: mikeb, Cmd: database.Set, } readMike := &database.TxEntry{ Bucket: ub, Key: []byte("mike"), Cmd: database.Get, } setMariano := &database.TxEntry{ Bucket: ub, Key: []byte("mariano"), Value: marianob, Cmd: database.Set, } setSeb := &database.TxEntry{ Bucket: ub, Key: []byte("sebastian"), Value: sebb, Cmd: database.Set, } readSeb := &database.TxEntry{ Bucket: ub, Key: []byte("sebastian"), Cmd: database.Get, } casGates2 := &database.TxEntry{ Bucket: ub, Key: []byte("bill"), Value: boogers, CmpValue: gatesb, Cmd: database.CmpAndSwap, } casGates3 := &database.TxEntry{ Bucket: ub, Key: []byte("bill"), Value: []byte("belly-button-lint"), CmpValue: gatesb, Cmd: database.CmpAndSwap, } // update: read write multiple entries. tx := &database.Tx{Operations: []*database.TxEntry{setMike, setMariano, readMike, setSeb, readSeb, casGates, casGates2, casGates3}} assert.Nil(t, db.Update(tx)) // verify that mike is in db res, err = db.Get(ub, []byte("mike")) assert.FatalError(t, err) assert.Equals(t, mikeb, res) // verify that mariano is in db res, err = db.Get(ub, []byte("mariano")) assert.FatalError(t, err) assert.Equals(t, marianob, res) // verify that bill gates is in db res, err = db.Get(ub, []byte("bill")) assert.FatalError(t, err) assert.Equals(t, boogers, res) // verify that seb is in db res, err = db.Get(ub, []byte("sebastian")) assert.FatalError(t, err) assert.Equals(t, sebb, res) // check that the readMike update txn was successful assert.Equals(t, readMike.Result, mikeb) // check that the readSeb update txn was successful assert.Equals(t, readSeb.Result, sebb) // check that the casGates update txn was a successful write assert.True(t, casGates.Swapped) assert.Equals(t, casGates.Result, gatesb) // check that the casGates2 update txn was successful assert.True(t, casGates2.Swapped) assert.Equals(t, casGates2.Result, boogers) // check that the casGates3 update txn was did not update. assert.False(t, casGates3.Swapped) assert.Equals(t, casGates3.Result, boogers) // List // _, err = db.List([]byte("clever")) assert.True(t, IsErrNotFound(err)) entries, err = db.List(ub) assert.FatalError(t, err) assert.Equals(t, len(entries), 4) // Update Again // // create txns for update test max := testUser{"max", "furman", 6} maxb, err := json.Marshal(max) assert.FatalError(t, err) maxey := testUser{"mike", "maxey", 3} maxeyb, err := json.Marshal(maxey) assert.FatalError(t, err) delMike := &database.TxEntry{ Bucket: ub, Key: []byte("mike"), Cmd: database.Delete, } setMax := &database.TxEntry{ Bucket: ub, Key: []byte("max"), Value: maxb, Cmd: database.Set, } setMaxey := &database.TxEntry{ Bucket: ub, Key: []byte("maxey"), Value: maxeyb, Cmd: database.Set, } delMaxey := &database.TxEntry{ Bucket: ub, Key: []byte("maxey"), Cmd: database.Delete, } delSeb := &database.TxEntry{ Bucket: ub, Key: []byte("sebastian"), Cmd: database.Delete, } // update: read write multiple entries. tx = &database.Tx{Operations: []*database.TxEntry{ delMike, setMax, setMaxey, delMaxey, delSeb, }} assert.Nil(t, db.Update(tx)) entries, err = db.List(ub) assert.FatalError(t, err) assert.Equals(t, len(entries), 3) // verify that max and mariano are in the db res, err = db.Get(ub, []byte("max")) assert.FatalError(t, err) assert.Equals(t, maxb, res) res, err = db.Get(ub, []byte("mariano")) assert.FatalError(t, err) assert.Equals(t, marianob, res) assert.Nil(t, db.DeleteTable(ub)) _, err = db.List(ub) assert.True(t, IsErrNotFound(err)) } func TestMain(m *testing.M) { // setup path := "./tmp" if _, err := os.Stat(path); os.IsNotExist(err) { os.Mkdir(path, 0755) } // run ret := m.Run() // teardown os.RemoveAll(path) os.Exit(ret) } func TestMySQL(t *testing.T) { var ( uname = "travis" pwd = "" proto = "tcp" addr = "127.0.0.1:3306" //path = "/tmp/mysql.sock" testDB = "test" ) isTravisTest := os.Getenv("TRAVIS") if len(isTravisTest) == 0 { fmt.Printf("Not running MySql integration tests\n") return } db, err := New("mysql", //fmt.Sprintf("%s:%s@%s(%s)/", uname, pwd, proto, addr), fmt.Sprintf("%s:%s@%s(%s)/", uname, pwd, proto, addr), WithDatabase(testDB)) assert.FatalError(t, err) defer db.Close() run(t, db) } func TestBadger(t *testing.T) { path := "./tmp/badgerdb" if _, err := os.Stat(path); os.IsNotExist(err) { assert.FatalError(t, os.Mkdir(path, 0755)) } db, err := New("badger", path, WithValueDir(path)) assert.FatalError(t, err) defer db.Close() run(t, db) } func TestBolt(t *testing.T) { assert.FatalError(t, os.MkdirAll("./tmp", 0644)) db, err := New("bbolt", "./tmp/boltdb") assert.FatalError(t, err) defer db.Close() run(t, db) }