")
assert.NotNil(t, err)
}
}
toolbox-0.33.2/mime_type.go 0000664 0000000 0000000 00000001114 13741102511 0015574 0 ustar 00root root 0000000 0000000 package toolbox
const (
//JSONMimeType JSON mime type constant
JSONMimeType = "text/json"
//CSVMimeType csv mime type constant
CSVMimeType = "text/csv"
//TSVMimeType tab separated mime type constant
TSVMimeType = "text/tsv"
//TextMimeType mime type constant
TextMimeType = "text/sql"
)
//FileExtensionMimeType json, csv, tsc, sql mime types.
var FileExtensionMimeType = map[string]string{
"json": JSONMimeType,
"csv": CSVMimeType,
"tsv": TSVMimeType,
"sql": TextMimeType,
"html": "text/html",
"js": "text/javascript",
"jpg": "image/jpeg",
"png": "image/png",
}
toolbox-0.33.2/os_helper.go 0000664 0000000 0000000 00000002224 13741102511 0015567 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"os"
"path"
"strings"
)
var dirMode os.FileMode = 0744
// RemoveFileIfExist remove file if exists
func RemoveFileIfExist(filenames ...string) error {
for _, filename := range filenames {
if !FileExists(filename) {
continue
}
err := os.Remove(filename)
if err != nil {
return err
}
}
return nil
}
// FileExists checks if file exists
func FileExists(filename string) bool {
if _, err := os.Stat(filename); err != nil {
return false
}
return true
}
// IsDirectory checks if file is directory
func IsDirectory(location string) bool {
if stat, _ := os.Stat(location); stat != nil {
return stat.IsDir()
}
return false
}
// CreateDirIfNotExist creates directory if they do not exist
func CreateDirIfNotExist(dirs ...string) error {
for _, dir := range dirs {
if len(dir) > 1 && strings.HasSuffix(dir, "/") {
dir = dir[:len(dir)-1]
}
parent, _ := path.Split(dir)
if parent != "/" && parent != dir {
CreateDirIfNotExist(parent)
}
if !FileExists(dir) {
err := os.Mkdir(dir, dirMode)
if err != nil {
return fmt.Errorf("failed to create dir %v %v", dir, err)
}
}
}
return nil
}
toolbox-0.33.2/os_helper_test.go 0000664 0000000 0000000 00000000546 13741102511 0016633 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"testing"
)
func TestCreateDirIfNotExist(t *testing.T) {
dir := "/tmp/abc"
toolbox.RemoveFileIfExist(dir)
toolbox.CreateDirIfNotExist(dir)
assert.True(t, toolbox.FileExists(dir))
toolbox.RemoveFileIfExist(dir)
assert.False(t, toolbox.FileExists(dir))
}
toolbox-0.33.2/predicate.go 0000664 0000000 0000000 00000000235 13741102511 0015547 0 ustar 00root root 0000000 0000000 package toolbox
//Predicate represents a generic predicate
type Predicate interface {
//Apply checks if predicate is true
Apply(value interface{}) bool
}
toolbox-0.33.2/predicates.go 0000664 0000000 0000000 00000012673 13741102511 0015743 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"reflect"
"strings"
"time"
)
//TrueProvider represents a true provider
var TrueProvider = func(input interface{}) bool {
return true
}
type withinSecPredicate struct {
baseTime time.Time
deltaInSeconds int
dateLayout string
actual string
elapsed time.Duration
maxAllowedDelay time.Duration
}
func (p *withinSecPredicate) String() string {
return fmt.Sprintf("(elapsed: %d, max allowed delay: %d)\n", int(p.elapsed), int(p.maxAllowedDelay))
}
//Apply returns true if passed in time is within deltaInSeconds from baseTime
func (p *withinSecPredicate) Apply(value interface{}) bool {
timeValue, err := ToTime(value, p.dateLayout)
if err != nil {
return false
}
elapsed := timeValue.Sub(p.baseTime)
if elapsed < 0 {
elapsed *= -1
}
var maxAllowedDelay = time.Duration(p.deltaInSeconds) * time.Second
var passed = maxAllowedDelay >= elapsed
if !passed {
p.elapsed = elapsed
p.maxAllowedDelay = maxAllowedDelay
}
return passed
}
func (p *withinSecPredicate) ToString() string {
return fmt.Sprintf(" %v within %v s", p.baseTime, p.deltaInSeconds)
}
//NewWithinPredicate returns new NewWithinPredicate predicate, it takes base time, delta in second, and dateLayout
func NewWithinPredicate(baseTime time.Time, deltaInSeconds int, dateLayout string) Predicate {
return &withinSecPredicate{
baseTime: baseTime,
deltaInSeconds: deltaInSeconds,
dateLayout: dateLayout,
}
}
type betweenPredicate struct {
from float64
to float64
}
func (p *betweenPredicate) Apply(value interface{}) bool {
floatValue := AsFloat(value)
return floatValue >= p.from && floatValue <= p.to
}
func (p *betweenPredicate) String() string {
return fmt.Sprintf("x BETWEEN %v AND %v", p.from, p.to)
}
//NewBetweenPredicate creates a new BETWEEN predicate, it takes from, and to.
func NewBetweenPredicate(from, to interface{}) Predicate {
return &betweenPredicate{
from: AsFloat(from),
to: AsFloat(to),
}
}
type inPredicate struct {
predicate Predicate
}
func (p *inPredicate) Apply(value interface{}) bool {
return p.predicate.Apply(value)
}
//NewInPredicate creates a new IN predicate
func NewInPredicate(values ...interface{}) Predicate {
converted, kind := DiscoverCollectionValuesAndKind(values)
switch kind {
case reflect.Int:
predicate := inIntPredicate{values: make(map[int]bool)}
SliceToMap(converted, predicate.values, func(item interface{}) int {
return AsInt(item)
}, TrueProvider)
return &predicate
case reflect.Float64:
predicate := inFloatPredicate{values: make(map[float64]bool)}
SliceToMap(converted, predicate.values, func(item interface{}) float64 {
return AsFloat(item)
}, TrueProvider)
return &predicate
default:
predicate := inStringPredicate{values: make(map[string]bool)}
SliceToMap(converted, predicate.values, func(item interface{}) string {
return AsString(item)
}, TrueProvider)
return &predicate
}
}
type inFloatPredicate struct {
values map[float64]bool
}
func (p *inFloatPredicate) Apply(value interface{}) bool {
candidate := AsFloat(value)
return p.values[candidate]
}
type inIntPredicate struct {
values map[int]bool
}
func (p *inIntPredicate) Apply(value interface{}) bool {
candidate := AsInt(value)
return p.values[int(candidate)]
}
type inStringPredicate struct {
values map[string]bool
}
func (p *inStringPredicate) Apply(value interface{}) bool {
candidate := AsString(value)
return p.values[candidate]
}
type numericComparablePredicate struct {
rightOperand float64
operator string
}
func (p *numericComparablePredicate) Apply(value interface{}) bool {
leftOperand := AsFloat(value)
switch p.operator {
case ">":
return leftOperand > p.rightOperand
case ">=":
return leftOperand >= p.rightOperand
case "<":
return leftOperand < p.rightOperand
case "<=":
return leftOperand <= p.rightOperand
case "=":
return leftOperand == p.rightOperand
case "!=":
return leftOperand != p.rightOperand
}
return false
}
type stringComparablePredicate struct {
rightOperand string
operator string
}
func (p *stringComparablePredicate) Apply(value interface{}) bool {
leftOperand := AsString(value)
switch p.operator {
case "=":
return leftOperand == p.rightOperand
case "!=":
return leftOperand != p.rightOperand
}
return false
}
//NewComparablePredicate create a new comparable predicate for =, !=, >=, <=
func NewComparablePredicate(operator string, leftOperand interface{}) Predicate {
if CanConvertToFloat(leftOperand) {
return &numericComparablePredicate{AsFloat(leftOperand), operator}
}
return &stringComparablePredicate{AsString(leftOperand), operator}
}
type nilPredicate struct{}
func (p *nilPredicate) Apply(value interface{}) bool {
return value == nil || reflect.ValueOf(value).IsNil()
}
//NewNilPredicate returns a new nil predicate
func NewNilPredicate() Predicate {
return &nilPredicate{}
}
type likePredicate struct {
matchingFragments []string
}
func (p *likePredicate) Apply(value interface{}) bool {
textValue := strings.ToLower(AsString(value))
for _, matchingFragment := range p.matchingFragments {
matchingIndex := strings.Index(textValue, matchingFragment)
if matchingIndex == -1 {
return false
}
if matchingIndex < len(textValue) {
textValue = textValue[matchingIndex:]
}
}
return true
}
//NewLikePredicate create a new like predicate
func NewLikePredicate(matching string) Predicate {
return &likePredicate{matchingFragments: strings.Split(strings.ToLower(matching), "%")}
}
toolbox-0.33.2/predicates_test.go 0000664 0000000 0000000 00000004772 13741102511 0017003 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"testing"
"time"
)
func TestWithinPredicate(t *testing.T) {
targetTime := time.Unix(1465009041, 0)
predicate := toolbox.NewWithinPredicate(targetTime, 2, "")
timeValue := time.Unix(1465009042, 0)
assert.True(t, predicate.Apply(timeValue))
timeValue = time.Unix(1465009044, 0)
assert.False(t, predicate.Apply(timeValue))
}
func TestBetweenPredicate(t *testing.T) {
predicate := toolbox.NewBetweenPredicate(10, 20)
assert.False(t, predicate.Apply(9))
assert.True(t, predicate.Apply(10))
assert.True(t, predicate.Apply(11))
assert.False(t, predicate.Apply(21))
}
func TestInPredicate(t *testing.T) {
{
predicate := toolbox.NewInPredicate("10", 20, "a")
assert.False(t, predicate.Apply(9))
assert.True(t, predicate.Apply(10))
assert.False(t, predicate.Apply(15))
assert.True(t, predicate.Apply("a"))
assert.True(t, predicate.Apply(20))
assert.False(t, predicate.Apply(21))
}
}
func TestComparablePredicate(t *testing.T) {
{
predicate := toolbox.NewComparablePredicate(">", "1")
assert.True(t, predicate.Apply(3))
assert.False(t, predicate.Apply(1))
}
{
predicate := toolbox.NewComparablePredicate("<", "1")
assert.True(t, predicate.Apply(0))
assert.False(t, predicate.Apply(3))
}
{
predicate := toolbox.NewComparablePredicate("!=", "1")
assert.True(t, predicate.Apply(0))
assert.False(t, predicate.Apply(1))
}
}
func TestNewLikePredicate(t *testing.T) {
{
predicate := toolbox.NewLikePredicate("abc%efg")
assert.False(t, predicate.Apply("abefg"))
assert.True(t, predicate.Apply("abcefg"))
}
{
predicate := toolbox.NewLikePredicate("abc%")
assert.True(t, predicate.Apply("abcfg"))
}
}
func TestNewComparablePredicate(t *testing.T) {
{
predicate := toolbox.NewComparablePredicate("=", "abc")
assert.True(t, predicate.Apply("abc"))
assert.False(t, predicate.Apply("abdc"))
}
{
predicate := toolbox.NewComparablePredicate("!=", "abc")
assert.True(t, predicate.Apply("abcc"))
assert.False(t, predicate.Apply("abc"))
}
{
predicate := toolbox.NewComparablePredicate(">=", 3)
assert.True(t, predicate.Apply(10))
assert.False(t, predicate.Apply(1))
}
{
predicate := toolbox.NewComparablePredicate("<=", 3)
assert.True(t, predicate.Apply(1))
assert.False(t, predicate.Apply(10))
}
}
func TestNewInPredicate(t *testing.T) {
predicate := toolbox.NewInPredicate(1.2, 1.5)
assert.True(t, predicate.Apply("1.2"))
assert.False(t, predicate.Apply("1.1"))
}
toolbox-0.33.2/ranger.go 0000664 0000000 0000000 00000000255 13741102511 0015067 0 ustar 00root root 0000000 0000000 package toolbox
//Ranger represents an abstraction that has ability range over collection item
type Ranger interface {
Range(func(item interface{}) (bool, error)) error
}
toolbox-0.33.2/sampler/ 0000775 0000000 0000000 00000000000 13741102511 0014723 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/sampler/service.go 0000664 0000000 0000000 00000001577 13741102511 0016724 0 ustar 00root root 0000000 0000000 package sampler
import (
"math/rand"
"sync"
"time"
)
//Service represents a sampler service
type Service struct {
rand *rand.Rand
acceptThreshold int32
mux sync.Mutex
PCT float64
}
//Accept accept sample meeting accept gaol PCT
func (s *Service) Accept() bool {
s.mux.Lock()
defer s.mux.Unlock()
n := s.rand.Int31n(100000)
return n < s.acceptThreshold
}
//Accept accept with threshold with PCT being ignored (same as PCT=100)
func (s *Service) AcceptWithThreshold(threshold float64) bool {
s.mux.Lock()
defer s.mux.Unlock()
n := s.rand.Int31n(100000)
return n < int32(threshold*1000.0)
}
//New creates a pct sampler
func New(acceptPCT float64) *Service {
source := rand.NewSource(time.Now().UnixNano())
return &Service{
PCT: acceptPCT,
rand: rand.New(source),
acceptThreshold: int32(acceptPCT * 1000.0),
}
}
toolbox-0.33.2/sampler/service_test.go 0000664 0000000 0000000 00000002657 13741102511 0017763 0 ustar 00root root 0000000 0000000 package sampler
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestService_Accept(t *testing.T) {
useCases := []struct {
description string
goalPCT float64
testCount int
}{
{
description: "50% goal",
goalPCT: 50,
testCount: 100000,
},
{
description: "95.1% goal",
goalPCT: 95.1,
testCount: 100000,
},
}
for _, useCase := range useCases {
sampler := New(useCase.goalPCT)
acceptCount := 0
for i := 0; i < useCase.testCount; i++ {
if sampler.Accept() {
acceptCount++
}
}
actualAcceptPCT := int(100.0 * (float64(acceptCount) / float64(useCase.testCount)))
if actualAcceptPCT > int(useCase.goalPCT) { //allows -1 diff
actualAcceptPCT--
}
if actualAcceptPCT < int(useCase.goalPCT) { //allows +1 diff
actualAcceptPCT++
}
assert.Equal(t, int(useCase.goalPCT), actualAcceptPCT, useCase.description)
}
}
func TestService_AcceptWithThreshold(t *testing.T) {
sampler := New(100) // should not affect the result
acceptCount := 0
testcount := 100000
for i := 0; i < testcount; i++ {
if sampler.AcceptWithThreshold(50.0) {
acceptCount++
}
}
actualAcceptPCT := int(100.0 * (float64(acceptCount) / float64(testcount)))
if actualAcceptPCT > int(50.0) { //allows -1 diff
actualAcceptPCT--
}
if actualAcceptPCT < int(50.0) { //allows +1 diff
actualAcceptPCT++
}
assert.Equal(t, int(50), actualAcceptPCT, "accept with threshold off ")
}
toolbox-0.33.2/secret/ 0000775 0000000 0000000 00000000000 13741102511 0014545 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/secret/README.md 0000664 0000000 0000000 00000007004 13741102511 0016025 0 ustar 00root root 0000000 0000000 ## Secret service ##
Secret service provide convenient way of handling credentials.
Service uses [credential config](./../cred/config.go) to store various provided credentials.
## Credentials retrieval
Service supports the following form of **secret** to retrieve credentials:
1. URL i.e. mem://secret/localhost.json
2. Relative path i.e. localhost.json, in this case based directory will be used to lookup credential resource
3. Short name i.e. localhost, in this case based directory will be used to lookup credential resource and .json ext will be added.
4. Inline certificates or secrets that does not represent resource location are placed into cred.Config.Data
**Base directory** can be file or URL, if empty '$HOME/.secret/' is used
```go
service := New(baseDirectory, false)
var secret = "localhost"
service.GetCredentials(secret)
```
## Credentials generation
Service allows for interactive credential generation, in this scenario services asks
a user for username and password for supplied secret.
Optionally private key path can be supplied for pub key based auth.
```go
privateKeyPath := ""//optional path
secret := "localhost"
service := New(baseDirectory, false)
location, err := service.Create(secret, privateKeyPath)
```
The following example uses service in the interactive mode, which will try to lookup credential
for supplied key, if failed it will ask a user for credential in terminal.
```go
service := New(baseDirectory, true)
config, err := service.GetOrCreate("xxxx")
```
## Secret expansion
Very common case for the application it to take encrypted credential to used wither username or password.
For example while running terminal command we may need to provide super user password and sometimes other secret,
in one command that we do not want to reveal to final user.
Take the following code as example:
```go
service := New(baseDirectory, true)
secrets := NewSecrets()
{//password expansion
secrets["mysql"] = "~/.secret/mysql.json"
input := "docker run --name db1 -e MYSQL_ROOT_PASSWORD=${mysql.password} -d mysql:tag"
expaned, err := service.Expand(input, secrets)
}
{//username and password expansion
secrets["pg"] = "~/.secret/pg.json"
input := "docker run --name some-postgres -e POSTGRES_PASSWORD=${pg.password} -e POSTGRES_USER=${pg.username} -d postgres"
expaned, err := service.Expand(input, secrets)
}
```
Secrets represents credentials secret map defined as `type Secrets map[SecretKey]Secret`
Here are some possible combination of secret map pairs.
```json
{
"git": "${env.HOME}/.secret/git.json",
"github.com": "${env.HOME}/.secret/github.json",
"github.private.com": "${env.HOME}/.secret/github-private.json",
"**replace**": "${env.HOME}/.secret/git.json"
}
```
The secret key can be static or dynamic. The first type in input/command is enclosed with either '*' or '#', the later is not.
In the command corresponding dynamic key can be enclosed with the following
1) '${secretKey.password}' for password expansion i.e. command: **${git.password}** will expand to password from git secret key
2) '**' for password expansion i.e. command: `**git**`will expand to password from git secret key
3) '${secretKey.username}' for username expansion i.e. command: **${git.username}** will expand to username from git secret key
4) '##' for username expansion i.e. command: `##git##` will expand to username from git secret key
toolbox-0.33.2/secret/helper.go 0000664 0000000 0000000 00000002503 13741102511 0016353 0 ustar 00root root 0000000 0000000 package secret
import (
"bufio"
"errors"
"fmt"
"golang.org/x/crypto/ssh/terminal"
"os"
"strings"
"syscall"
"time"
// "github.com/bgentry/speakeasy"
)
//ReadingCredentialTimeout represents max time for providing CredentialsFromLocation
var ReadingCredentialTimeout = time.Second * 45
var ReadUserAndPassword = func(timeout time.Duration) (user string, pass string, err error) {
completed := make(chan bool)
var reader = func() {
defer func() {
completed <- true
}()
var bytePassword, bytePassword2 []byte
reader := bufio.NewReader(os.Stdin)
fmt.Print("Enter Username: ")
user, _ = reader.ReadString('\n')
fmt.Print("Enter Password: ")
bytePassword, err = terminal.ReadPassword(int(syscall.Stdin))
if err != nil {
err = fmt.Errorf("failed to read password %v", err)
return
}
fmt.Print("\nRetype Password: ")
bytePassword2, err = terminal.ReadPassword(int(syscall.Stdin))
if err != nil {
err = fmt.Errorf("failed to read password %v", err)
return
}
password := string(bytePassword)
if string(bytePassword2) != password {
err = errors.New("password did not match")
}
}
go reader()
select {
case <-completed:
case <-time.After(timeout):
err = fmt.Errorf("reading credential timeout")
}
user = strings.TrimSpace(user)
pass = strings.TrimSpace(pass)
return user, pass, err
}
toolbox-0.33.2/secret/helper_test.go 0000664 0000000 0000000 00000000203 13741102511 0017405 0 ustar 00root root 0000000 0000000 package secret
import (
"testing"
"time"
)
func TestSecretKey_Secret(t *testing.T) {
ReadUserAndPassword(time.Second * 10)
}
toolbox-0.33.2/secret/secret.go 0000664 0000000 0000000 00000003715 13741102511 0016367 0 ustar 00root root 0000000 0000000 package secret
import (
"github.com/viant/toolbox/cred"
"strings"
)
//Secrets represents CredentialsFromLocation secret map
type Secrets map[SecretKey]Secret
//NewSecrets creates new secrets
func NewSecrets(secrets map[string]string) Secrets {
var result = make(map[SecretKey]Secret)
if len(secrets) == 0 {
return result
}
for k, v := range secrets {
result[SecretKey(k)] = Secret(v)
}
return result
}
/**
SecretKey represent secret key
Take the following secrets as example:
"secrets": {
"git": "${env.HOME}/.secret/git.json",
"github.com": "${env.HOME}/.secret/github.json",
"github.private.com": "${env.HOME}/.secret/github-private.json",
"**replace**": "${env.HOME}/.secret/git.json",
}
The secret key can be static or dynamic. The first type is already enclosed with '*' or '#', the later is not.
In the command corresponding dynamic key can be enclosed with the following
'**' for password expansion i.e. command: **git** will expand to password from git secret key
'##' for username expansion i.e. command: ##git## will expand to username from git secret key
*/
type SecretKey string
//IsDynamic returns true if key is dynamic
func (s SecretKey) IsDynamic() bool {
return !(strings.HasPrefix(string(s), "*") || strings.HasPrefix(string(s), "#"))
}
//String returns secret key as string
func (s SecretKey) String() string {
return string(s)
}
//Get extracts username or password or JSON based on key type (# prefix for user, otherwise password or JSON)
func (s SecretKey) Secret(cred *cred.Config) string {
if strings.HasPrefix(s.String(), "#") || strings.HasSuffix(s.String(), ".username}") || strings.HasSuffix(s.String(), ".Username}") {
return cred.Username
}
if cred.Password != "" {
return cred.Password
}
return cred.Data
}
//Secret represents a secret
type Secret string
//IsLocation returns true if secret is a location
func (s Secret) IsLocation() bool {
return !strings.ContainsAny(string(s), "{}[]=+()@#^&*|")
}
toolbox-0.33.2/secret/secret_test.go 0000664 0000000 0000000 00000002243 13741102511 0017421 0 ustar 00root root 0000000 0000000 package secret_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/secret"
"testing"
)
func TestSecret_IsLocation(t *testing.T) {
{
sec := secret.Secret("htttp://abc.com/secret.json")
assert.True(t, sec.IsLocation())
}
{
sec := secret.Secret("{}")
assert.False(t, sec.IsLocation())
}
}
func TestSecretKey_IsDynamic(t *testing.T) {
{
key := secret.SecretKey("**key1**")
assert.False(t, key.IsDynamic())
}
{
key := secret.SecretKey("##key1##")
assert.False(t, key.IsDynamic())
}
{
key := secret.SecretKey("key1")
assert.True(t, key.IsDynamic())
}
}
func TestSecretKey_Secret(t *testing.T) {
{ //test user and password secret
credConfig := &cred.Config{Username: "abc", Password: "pass"}
{
key := secret.SecretKey("**key1**")
assert.EqualValues(t, "pass", key.Secret(credConfig))
}
{
key := secret.SecretKey("##key1##")
assert.EqualValues(t, "abc", key.Secret(credConfig))
}
}
{ //test json secret
credConfig := &cred.Config{Username: "abc", Data: "{}"}
{
key := secret.SecretKey("**key1**")
assert.EqualValues(t, "{}", key.Secret(credConfig))
}
}
}
toolbox-0.33.2/secret/service.go 0000664 0000000 0000000 00000013564 13741102511 0016545 0 ustar 00root root 0000000 0000000 package secret
import (
"bytes"
"errors"
"fmt"
"github.com/viant/toolbox"
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/data"
"github.com/viant/toolbox/storage"
"github.com/viant/toolbox/url"
"os"
"path"
"strings"
"sync"
)
//represents a secret service
type Service struct {
interactive bool
baseDirectory string
cache map[string]*cred.Config
lock *sync.RWMutex
}
func (s *Service) CredentialsLocation(secret string) (string, error) {
if secret == "" {
return "", errors.New("secretLocation was empty")
}
if path.Ext(secret) == "" {
secret += ".json"
}
if strings.HasPrefix(secret, "~") {
secret = strings.Replace(secret, "~", os.Getenv("HOME"), 1)
}
currentDirectory, _ := os.Getwd()
for _, candidate := range []string{secret, toolbox.URLPathJoin(currentDirectory, secret)} {
if toolbox.FileExists(candidate) {
return candidate, nil
}
}
if strings.Contains(secret, ":/") {
return secret, nil
}
return toolbox.URLPathJoin(s.baseDirectory, secret), nil
}
//Credentials returns credential config for supplied location.
func (s *Service) CredentialsFromLocation(secret string) (*cred.Config, error) {
secretLocation, err := s.CredentialsLocation(secret)
if err != nil {
return nil, err
}
s.lock.RLock()
credConfig, has := s.cache[secretLocation]
s.lock.RUnlock()
if has {
return credConfig, nil
}
resource := url.NewResource(secretLocation)
configContent, err := resource.Download()
if err != nil {
return nil, fmt.Errorf("failed to open: '%v', due to: %v", secretLocation, err)
}
credConfig = &cred.Config{}
if err = credConfig.LoadFromReader(bytes.NewReader(configContent), path.Ext(secretLocation)); err != nil {
return nil, err
}
credConfig.Data = string(configContent)
s.lock.Lock()
defer s.lock.Unlock()
s.cache[secretLocation] = credConfig
return credConfig, nil
}
//GetOrCreate gets or creates credential
func (s *Service) GetOrCreate(secret string) (*cred.Config, error) {
if secret == "" {
return nil, errors.New("secret was empty")
}
result, err := s.GetCredentials(secret)
if err != nil && s.interactive && Secret(secret).IsLocation() {
secretLocation, err := s.Create(secret, "")
if err != nil {
return nil, err
}
return s.GetCredentials(secretLocation)
}
return result, err
}
//Credentials returns credential config
func (s *Service) GetCredentials(secret string) (*cred.Config, error) {
if !Secret(secret).IsLocation() {
var result = &cred.Config{Data: string(secret)}
//try to load credential
err := result.LoadFromReader(strings.NewReader(string(secret)), "")
return result, err
}
return s.CredentialsFromLocation(secret)
}
func (s *Service) expandDynamicSecret(input string, key SecretKey, secret Secret) (string, error) {
if !strings.Contains(input, key.String()) {
return input, nil
}
credConfig, err := s.GetOrCreate(string(secret))
if err != nil {
return "", err
}
createMap := data.NewMap()
credInfo := map[string]interface{}{}
_ = toolbox.DefaultConverter.AssignConverted(&credInfo, credInfo)
credInfo["username"] = credConfig.Username
credInfo["password"] = credConfig.Password
createMap.Put(string(key), credInfo)
passwordKey := fmt.Sprintf("**%v**", key)
if count := strings.Count(input, passwordKey); count > 0 {
secret := credConfig.Password
if secret == "" {
secret = credConfig.Data
}
input = strings.Replace(input, passwordKey, secret, count)
}
userKey := fmt.Sprintf("##%v##", key)
if count := strings.Count(input, userKey); count > 0 {
input = strings.Replace(input, userKey, credConfig.Username, count)
}
if index := strings.Index(input, "$"); index == -1 {
return input, nil
}
return createMap.ExpandAsText(input), nil
}
func (s *Service) expandSecret(command string, key SecretKey, secret Secret) (string, error) {
credConfig, err := s.GetOrCreate(string(secret))
if err != nil {
return "", err
}
command = strings.Replace(command, key.String(), key.Secret(credConfig), 1)
return command, nil
}
//Expand expands input credential keys with actual CredentialsFromLocation
func (s *Service) Expand(input string, credentials map[SecretKey]Secret) (string, error) {
if len(credentials) == 0 {
return input, nil
}
var err error
for k, v := range credentials {
if strings.Contains(input, k.String()) {
if k.IsDynamic() {
input, err = s.expandDynamicSecret(input, k, v)
} else {
input, err = s.expandSecret(input, k, v)
}
if err != nil {
return "", err
}
}
}
return input, nil
}
//Create creates a new credential config for supplied name
func (s *Service) Create(name, privateKeyPath string) (string, error) {
if strings.HasPrefix(privateKeyPath, "~") {
privateKeyPath = strings.Replace(privateKeyPath, "~", os.Getenv("HOME"), 1)
}
username, password, err := ReadUserAndPassword(ReadingCredentialTimeout)
if err != nil {
return "", err
}
config := &cred.Config{
Username: username,
Password: password,
}
if toolbox.FileExists(privateKeyPath) && !cred.IsKeyEncrypted(privateKeyPath) {
config.PrivateKeyPath = privateKeyPath
}
var secretResource = url.NewResource(toolbox.URLPathJoin(s.baseDirectory, fmt.Sprintf("%v.json", name)))
storageService, err := storage.NewServiceForURL(secretResource.URL, "")
if err != nil {
return "", err
}
buf := new(bytes.Buffer)
err = config.Write(buf)
if err != nil {
return "", err
}
err = storageService.Upload(secretResource.URL, buf)
return secretResource.URL, err
}
//NewSecretService creates a new secret service
func New(baseDirectory string, interactive bool) *Service {
if baseDirectory == "" {
baseDirectory = path.Join(os.Getenv("HOME"), ".secret")
} else if strings.HasPrefix(baseDirectory, "~") {
baseDirectory = strings.Replace(baseDirectory, "~", path.Join(os.Getenv("HOME"), ".secret"), 1)
}
return &Service{
baseDirectory: baseDirectory,
interactive: interactive,
cache: make(map[string]*cred.Config),
lock: &sync.RWMutex{},
}
}
toolbox-0.33.2/secret/service_test.go 0000664 0000000 0000000 00000012157 13741102511 0017601 0 ustar 00root root 0000000 0000000 package secret
import (
"bytes"
"errors"
"fmt"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/storage"
"testing"
"time"
)
func setupData(baseDirectory string, data map[string]*cred.Config) error {
storageService := storage.NewMemoryService()
for k, v := range data {
var buf = new(bytes.Buffer)
if err := v.Write(buf); err != nil {
return err
}
var URL = toolbox.URLPathJoin(baseDirectory, fmt.Sprintf("%v.json", k))
if err := storageService.Upload(URL, bytes.NewReader(buf.Bytes())); err != nil {
return err
}
}
return nil
}
func TestNew(t *testing.T) {
var baseDirectory = "mem://secret"
err := setupData(baseDirectory, map[string]*cred.Config{
"localhost": {Username: "user1", Password: "pass1"},
"10.3.3.12": {Username: "xxx", Password: "yyy"},
})
assert.Nil(t, err)
service := New(baseDirectory, false)
config, err := service.GetCredentials("localhost")
if assert.Nil(t, err) {
assert.EqualValues(t, "user1", config.Username)
assert.EqualValues(t, "pass1", config.Password)
}
_, err = service.GetCredentials("nonexisting")
assert.NotNil(t, err)
}
func TestService_Expand(t *testing.T) {
var baseDirectory = "mem://secret"
err := setupData(baseDirectory, map[string]*cred.Config{
"localhost": {Username: "user1", Password: "pass1"},
"10.3.3.12": {Username: "xxx", Data: `{"Key1":"abc"}`},
"github.com": {Username: "xxx", Password: "p"},
})
assert.Nil(t, err)
service := New(baseDirectory, false)
var original = ReadUserAndPassword
defer func() {
ReadUserAndPassword = original
}()
ReadUserAndPassword = func(timeout time.Duration) (string, string, error) {
return "user1", "password2", nil
}
var useCases = []struct {
Description string
Interactive bool
Matchable string
Input string
Credentials map[SecretKey]Secret
Expect string
HasError bool
}{
{
Description: "Password replacement with secret short name",
Input: "**sudo**",
Matchable: "",
Credentials: map[SecretKey]Secret{
"**sudo**": "localhost",
},
Expect: "pass1",
},
{
Description: "Password replacement with secret short name, explicit",
Input: "${sudo.password}",
Matchable: "",
Credentials: map[SecretKey]Secret{
"sudo": "localhost",
},
Expect: "pass1",
},
{
Description: "Username replacement with URL based secret",
Input: "##sudo##",
Matchable: "",
Credentials: map[SecretKey]Secret{
"##sudo##": "mem://secret/localhost",
},
Expect: "user1",
},
{
Description: "Username replacement with URL based secret and dynamic key",
Input: "##sudo##",
Matchable: "",
Credentials: map[SecretKey]Secret{
"sudo": "mem://secret/localhost",
},
Expect: "user1",
},
{
Description: "Username replacement with URL based secret and dynamic key - explicit",
Input: "${sudo.username}",
Matchable: "",
Credentials: map[SecretKey]Secret{
"sudo": "mem://secret/localhost",
},
Expect: "user1",
},
{
Description: "Non existing CredentialsFromLocation",
Input: "**key**",
Matchable: "",
Credentials: map[SecretKey]Secret{
"key": "abc",
},
HasError: true,
},
{
Description: "Interactive credential",
Interactive: true,
Input: "**key**",
Matchable: "",
Credentials: map[SecretKey]Secret{
"key": "test",
},
Expect: "password2",
},
}
for _, useCase := range useCases {
service.interactive = useCase.Interactive
_, err := service.Expand(useCase.Input, useCase.Credentials)
if useCase.HasError {
assert.NotNil(t, err, useCase.Description)
continue
}
if assert.Nil(t, err, useCase.Description) {
}
}
}
func TestService_Create(t *testing.T) {
service := New("mem://secret", false)
var original = ReadUserAndPassword
defer func() {
ReadUserAndPassword = original
}()
{ //read user and password
ReadUserAndPassword = func(timeout time.Duration) (string, string, error) {
return "user1", "password2", nil
}
location, err := service.Create("test", "")
assert.Nil(t, err)
assert.EqualValues(t, "mem://secret/test.json", location)
service := New("", false)
config, err := service.GetCredentials(location)
if assert.Nil(t, err) {
assert.EqualValues(t, "user1", config.Username)
assert.EqualValues(t, "password2", config.Password)
}
}
{ //Test interactive
service.interactive = true
ReadUserAndPassword = func(timeout time.Duration) (string, string, error) {
return "user2", "password3", nil
}
config, err := service.GetOrCreate("xxxx")
assert.Nil(t, err)
if assert.Nil(t, err) {
assert.EqualValues(t, "user2", config.Username)
assert.EqualValues(t, "password3", config.Password)
}
}
{ //read user and password with error
ReadUserAndPassword = func(timeout time.Duration) (string, string, error) {
return "", "", errors.New("test")
}
_, err := service.Create("test", "")
assert.NotNil(t, err)
}
}
func TestSecret_IsLocation(t *testing.T) {
secret := Secret("mem://github.com/viant/endly/workflow/docker/build/secret/build.json")
assert.True(t, secret.IsLocation())
}
toolbox-0.33.2/service_router.go 0000664 0000000 0000000 00000037707 13741102511 0016665 0 ustar 00root root 0000000 0000000 package toolbox
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"reflect"
"strings"
"time"
)
const (
jsonContentType = "application/json"
yamlContentTypeSuffix = "/yaml"
textPlainContentType = "text/plain"
contentTypeHeader = "Content-Type"
)
const (
//MethodGet HTTP GET meothd
MethodGet = "GET"
MethodHead = "HEAD"
MethodPost = "POST"
MethodPut = "PUT"
MethodPatch = "PATCH" // RFC 5789
MethodDelete = "DELETE"
MethodOptions = "OPTIONS"
MethodTrace = "TRACE"
)
var httpMethods = map[string]bool{
MethodDelete: true,
MethodGet: true,
MethodPatch: true,
MethodPost: true,
MethodPut: true,
MethodHead: true,
MethodTrace: true,
MethodOptions: true,
}
//HandlerInvoker method is responsible of passing required parameters to router handler.
type HandlerInvoker func(serviceRouting *ServiceRouting, request *http.Request, response http.ResponseWriter, parameters map[string]interface{}) error
//DefaultEncoderFactory - NewJSONEncoderFactory
var DefaultEncoderFactory = NewJSONEncoderFactory()
//DefaultDecoderFactory - NewJSONDecoderFactory
var DefaultDecoderFactory = NewJSONDecoderFactory()
//YamlDefaultEncoderFactory - NewYamlEncoderFactory
var YamlDefaultEncoderFactory = NewYamlEncoderFactory()
//YamlDefaultDecoderFactory - NewYamlDecoderFactory
var YamlDefaultDecoderFactory = NewFlexYamlDecoderFactory()
//ServiceRouting represents a simple web services routing rule, which is matched with http request
type ServiceRouting struct {
URI string //matching uri
Handler interface{} //has to be func
HTTPMethod string
Parameters []string
ContentTypeEncoders map[string]EncoderFactory //content type encoder factory
ContentTypeDecoders map[string]DecoderFactory //content type decoder factory
HandlerInvoker HandlerInvoker //optional function that will be used instead of reflection to invoke a handler.
}
func (sr ServiceRouting) getDecoderFactory(contentType string) DecoderFactory {
if sr.ContentTypeDecoders != nil {
if factory, found := sr.ContentTypeDecoders[contentType]; found {
return factory
}
}
if strings.HasSuffix(contentType, yamlContentTypeSuffix) {
return YamlDefaultDecoderFactory
}
return DefaultDecoderFactory
}
func (sr ServiceRouting) getEncoderFactory(contentType string) EncoderFactory {
if sr.ContentTypeDecoders != nil {
if factory, found := sr.ContentTypeEncoders[contentType]; found {
return factory
}
}
if strings.HasSuffix(contentType, yamlContentTypeSuffix) {
return YamlDefaultEncoderFactory
}
return DefaultEncoderFactory
}
func (sr ServiceRouting) extractParameterFromBody(parameterName string, targetType reflect.Type, request *http.Request) (interface{}, error) {
targetValuePointer := reflect.New(targetType)
contentType := getContentTypeOrJSONContentType(request.Header.Get(contentTypeHeader))
decoderFactory := sr.getDecoderFactory(contentType)
body, err := ioutil.ReadAll(request.Body)
if err != nil {
return nil, err
}
decoder := decoderFactory.Create(bytes.NewReader(body))
if !strings.Contains(parameterName, ":") {
err := decoder.Decode(targetValuePointer.Interface())
if err != nil {
return nil, fmt.Errorf("unable to extract %T due to: %v, body: !%s!", targetValuePointer.Interface(), err, body)
}
} else {
var valueMap = make(map[string]interface{})
pair := strings.SplitN(parameterName, ":", 2)
valueMap[pair[1]] = targetValuePointer.Interface()
err := decoder.Decode(&valueMap)
if err != nil {
return nil, fmt.Errorf("unable to extract %T due to %v", targetValuePointer.Interface(), err)
}
}
return targetValuePointer.Interface(), nil
}
func (sr ServiceRouting) extractParameters(request *http.Request, response http.ResponseWriter) (map[string]interface{}, error) {
var result = make(map[string]interface{})
_ = request.ParseForm()
functionSignature := GetFuncSignature(sr.Handler)
uriParameters, _ := ExtractURIParameters(sr.URI, request.RequestURI)
for _, name := range sr.Parameters {
value, found := uriParameters[name]
if found {
if strings.Contains(value, ",") {
result[name] = strings.Split(value, ",")
} else {
result[name] = value
}
continue
}
value = request.Form.Get(name)
if len(value) > 0 {
result[name] = value
} else {
continue
}
}
if HasSliceAnyElements(sr.Parameters, "@httpRequest") {
result["@httpRequest"] = request
}
if HasSliceAnyElements(sr.Parameters, "@httpResponseWriter") {
result["@httpResponseWriter"] = response
}
if request.ContentLength > 0 {
for i, parameter := range sr.Parameters {
if _, found := result[parameter]; !found {
value, err := sr.extractParameterFromBody(parameter, functionSignature[i], request)
if err != nil {
return nil, fmt.Errorf("failed to extract parameters for %v %v due to %v", sr.HTTPMethod, sr.URI, err)
}
result[parameter] = value
break
}
}
}
return result, nil
}
//ServiceRouter represents routing rule
type ServiceRouter struct {
serviceRouting []*ServiceRouting
}
func (r *ServiceRouter) match(request *http.Request) []*ServiceRouting {
var result = make([]*ServiceRouting, 0)
for _, candidate := range r.serviceRouting {
if candidate.HTTPMethod == request.Method {
_, matched := ExtractURIParameters(candidate.URI, request.RequestURI)
if matched {
result = append(result, candidate)
}
}
}
return result
}
func getContentTypeOrJSONContentType(contentType string) string {
if strings.Contains(contentType, textPlainContentType) || strings.Contains(contentType, jsonContentType) || contentType == "" {
return jsonContentType
}
return contentType
}
//Route matches service routing by http method , and number of parameters, then it call routing method, and sent back its response.
func (r *ServiceRouter) Route(response http.ResponseWriter, request *http.Request) error {
candidates := r.match(request)
if len(candidates) == 0 {
var uriTemplates = make([]string, 0)
for _, routing := range r.serviceRouting {
uriTemplates = append(uriTemplates, routing.URI)
}
return fmt.Errorf("failed to route request - unable to match %v with one of %v", request.RequestURI, strings.Join(uriTemplates, ","))
}
var finalError error
for _, serviceRouting := range candidates {
parameterValues, err := serviceRouting.extractParameters(request, response)
if err != nil {
finalError = fmt.Errorf("unable to extract parameters due to %v", err)
continue
}
if serviceRouting.HandlerInvoker != nil {
err := serviceRouting.HandlerInvoker(serviceRouting, request, response, parameterValues)
if err != nil {
finalError = fmt.Errorf("unable to extract parameters due to %v", err)
}
continue
}
functionParameters, err := BuildFunctionParameters(serviceRouting.Handler, serviceRouting.Parameters, parameterValues)
if err != nil {
finalError = fmt.Errorf("unable to build function parameters %T due to %v", serviceRouting.Handler, err)
continue
}
result := CallFunction(serviceRouting.Handler, functionParameters...)
if len(result) > 0 {
err = WriteServiceRoutingResponse(response, request, serviceRouting, result[0])
if err != nil {
return fmt.Errorf("failed to write response response %v, due to %v", result[0], err)
}
return nil
}
response.Header().Set(contentTypeHeader, textPlainContentType)
}
if finalError != nil {
return fmt.Errorf("failed to route request - %v", finalError)
}
return nil
}
//WriteServiceRoutingResponse writes service router response
func WriteServiceRoutingResponse(response http.ResponseWriter, request *http.Request, serviceRouting *ServiceRouting, result interface{}) error {
if result == nil {
result = struct{}{}
}
statusCodeAccessor, ok := result.(StatucCodeAccessor)
if ok {
statusCode := statusCodeAccessor.GetStatusCode()
if statusCode > 0 && statusCode != http.StatusOK {
response.WriteHeader(statusCode)
return nil
}
}
contentTypeAccessor, ok := result.(ContentTypeAccessor)
var responseContentType string
if ok {
responseContentType = contentTypeAccessor.GetContentType()
}
if responseContentType == "" {
requestContentType := request.Header.Get(contentTypeHeader)
responseContentType = getContentTypeOrJSONContentType(requestContentType)
}
encoderFactory := serviceRouting.getEncoderFactory(responseContentType)
encoder := encoderFactory.Create(response)
response.Header().Set(contentTypeHeader, responseContentType)
err := encoder.Encode(result)
if err != nil {
return fmt.Errorf("failed to encode response %v, due to %v", response, err)
}
return nil
if err != nil {
return fmt.Errorf("failed to write response response %v, due to %v", result, err)
}
return nil
}
//WriteResponse writes response to response writer, it used encoder factory to encode passed in response to the writer, it sets back request contenttype to response.
func (r *ServiceRouter) WriteResponse(encoderFactory EncoderFactory, response interface{}, request *http.Request, responseWriter http.ResponseWriter) error {
requestContentType := request.Header.Get(contentTypeHeader)
responseContentType := getContentTypeOrJSONContentType(requestContentType)
encoder := encoderFactory.Create(responseWriter)
responseWriter.Header().Set(contentTypeHeader, responseContentType)
err := encoder.Encode(response)
if err != nil {
return fmt.Errorf("failed to encode response %v, due to %v", response, err)
}
return nil
}
//NewServiceRouter creates a new service router, is takes list of service routing as arguments
func NewServiceRouter(serviceRouting ...ServiceRouting) *ServiceRouter {
var routings = make([]*ServiceRouting, 0)
for i := range serviceRouting {
routings = append(routings, &serviceRouting[i])
}
return &ServiceRouter{routings}
}
//RouteToService calls web service url, with passed in json request, and encodes http json response into passed response
func RouteToService(method, url string, request, response interface{}, options ...*HttpOptions) (err error) {
client, err := NewToolboxHTTPClient(options...)
if err != nil {
return err
}
return client.Request(method, url, request, response, NewJSONEncoderFactory(), NewJSONDecoderFactory())
}
type HttpOptions struct {
Key string
Value interface{}
}
func NewHttpClient(options ...*HttpOptions) (*http.Client, error) {
if len(options) == 0 {
return http.DefaultClient, nil
}
var (
// Default values matching DefaultHttpClient
RequestTimeoutMs = 30 * time.Second
KeepAliveTimeMs = 30 * time.Second
TLSHandshakeTimeoutMs = 10 * time.Second
ExpectContinueTimeout = 1 * time.Second
IdleConnTimeout = 90 * time.Second
DualStack = true
MaxIdleConnsPerHost = http.DefaultMaxIdleConnsPerHost
MaxIdleConns = 100
FollowRedirects = true
ResponseHeaderTimeoutMs time.Duration
TimeoutMs time.Duration
)
for _, option := range options {
switch option.Key {
case "RequestTimeoutMs":
RequestTimeoutMs = time.Duration(AsInt(option.Value)) * time.Millisecond
case "TimeoutMs":
TimeoutMs = time.Duration(AsInt(option.Value)) * time.Millisecond
case "KeepAliveTimeMs":
KeepAliveTimeMs = time.Duration(AsInt(option.Value)) * time.Millisecond
case "TLSHandshakeTimeoutMs":
KeepAliveTimeMs = time.Duration(AsInt(option.Value)) * time.Millisecond
case "ResponseHeaderTimeoutMs":
ResponseHeaderTimeoutMs = time.Duration(AsInt(option.Value)) * time.Millisecond
case "MaxIdleConns":
MaxIdleConns = AsInt(option.Value)
case "MaxIdleConnsPerHost":
MaxIdleConnsPerHost = AsInt(option.Value)
case "DualStack":
DualStack = AsBoolean(option.Value)
case "FollowRedirects":
FollowRedirects = AsBoolean(option.Value)
default:
return nil, fmt.Errorf("Invalid option: %v", option.Key)
}
}
roundTripper := http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: RequestTimeoutMs,
KeepAlive: KeepAliveTimeMs,
DualStack: DualStack,
}).DialContext,
MaxIdleConns: MaxIdleConns,
ExpectContinueTimeout: ExpectContinueTimeout,
IdleConnTimeout: IdleConnTimeout,
TLSHandshakeTimeout: TLSHandshakeTimeoutMs,
MaxIdleConnsPerHost: MaxIdleConnsPerHost,
ResponseHeaderTimeout: ResponseHeaderTimeoutMs,
}
client := &http.Client{
Transport: &roundTripper,
Timeout: TimeoutMs,
}
if !FollowRedirects {
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}
return client, nil
}
// ToolboxHTTPClient contains preconfigured http client
type ToolboxHTTPClient struct {
httpClient *http.Client
}
// NewToolboxHTTPClient instantiate new client with provided options
func NewToolboxHTTPClient(options ...*HttpOptions) (*ToolboxHTTPClient, error) {
client, err := NewHttpClient(options...)
if err != nil {
return nil, err
}
return &ToolboxHTTPClient{client}, nil
}
// Request sends http request using the existing client
func (c *ToolboxHTTPClient) Request(method, url string, request, response interface{}, encoderFactory EncoderFactory, decoderFactory DecoderFactory) (err error) {
if _, found := httpMethods[strings.ToUpper(method)]; !found {
return errors.New("unsupported method:" + method)
}
var buffer *bytes.Buffer
if request != nil {
buffer = new(bytes.Buffer)
if IsString(request) {
buffer.Write([]byte(AsString(request)))
} else {
err := encoderFactory.Create(buffer).Encode(&request)
if err != nil {
return fmt.Errorf("failed to encode request: %v due to ", err)
}
}
}
var serverResponse *http.Response
var httpRequest *http.Request
httpMethod := strings.ToUpper(method)
if request != nil {
httpRequest, err = http.NewRequest(httpMethod, url, buffer)
if err != nil {
return err
}
httpRequest.Header.Set(contentTypeHeader, jsonContentType)
} else {
httpRequest, err = http.NewRequest(httpMethod, url, nil)
if err != nil {
return err
}
}
serverResponse, err = c.httpClient.Do(httpRequest)
if serverResponse != nil {
// must close we have serverResponse to avoid fd leak
defer serverResponse.Body.Close()
}
if err != nil && serverResponse != nil {
return fmt.Errorf("failed to get response %v %v", err, serverResponse.Header.Get("error"))
}
if response != nil {
updateResponse(serverResponse, response)
if serverResponse == nil {
return fmt.Errorf("failed to receive response %v", err)
}
var errorPrefix = fmt.Sprintf("failed to process response: %v, ", serverResponse.StatusCode)
body, err := ioutil.ReadAll(serverResponse.Body)
if err != nil {
return fmt.Errorf("%v unable read body %v", errorPrefix, err)
}
if len(body) == 0 {
return fmt.Errorf("%v response body was empty", errorPrefix)
}
if serverResponse.StatusCode == http.StatusNotFound {
updateResponse(serverResponse, response)
return nil
}
if int(serverResponse.StatusCode/100)*100 == http.StatusInternalServerError {
return errors.New(string(body))
}
err = decoderFactory.Create(strings.NewReader(string(body))).Decode(response)
if err != nil {
return fmt.Errorf("%v. unable decode response as %T: body: %v: %v", errorPrefix, response, string(body), err)
}
updateResponse(serverResponse, response)
}
return nil
}
//StatucCodeMutator client side reponse optional interface
type StatucCodeMutator interface {
SetStatusCode(code int)
}
//StatucCodeAccessor server side response accessor
type StatucCodeAccessor interface {
GetStatusCode() int
}
//ContentTypeMutator client side reponse optional interface
type ContentTypeMutator interface {
SetContentType(contentType string)
}
//ContentTypeAccessor server side response accessor
type ContentTypeAccessor interface {
GetContentType() string
}
//updateResponse update response with content type and status code if applicable
func updateResponse(httpResponse *http.Response, response interface{}) {
if response == nil {
return
}
statusCodeMutator, ok := response.(StatucCodeMutator)
if ok {
statusCodeMutator.SetStatusCode(httpResponse.StatusCode)
}
contentTypeMutator, ok := response.(ContentTypeMutator)
if ok {
contentTypeMutator.SetContentType(httpResponse.Header.Get(contentTypeHeader))
}
}
toolbox-0.33.2/service_router_test.go 0000664 0000000 0000000 00000011007 13741102511 0017705 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"fmt"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"log"
"net/http"
"testing"
"time"
)
type ReverseService struct{}
func (this ReverseService) Reverse(values []int) []int {
var result = make([]int, 0)
for i := len(values) - 1; i >= 0; i-- {
result = append(result, values[i])
}
return result
}
func (this ReverseService) Reverse2(values []int) []int {
var result = make([]int, 0)
for i := len(values) - 1; i >= 0; i-- {
result = append(result, values[i])
}
return result
}
var ReverseInvoker = func(serviceRouting *toolbox.ServiceRouting, request *http.Request, response http.ResponseWriter, uriParameters map[string]interface{}) error {
var function = serviceRouting.Handler.(func(values []int) []int)
idsParam := uriParameters["ids"]
ids := idsParam.([]string)
values := make([]int, 0)
for _, item := range ids {
values = append(values, toolbox.AsInt(item))
}
var result = function(values)
err := toolbox.WriteServiceRoutingResponse(response, request, serviceRouting, result)
if err != nil {
return err
}
return nil
}
func StartServer(port string, t *testing.T) {
service := ReverseService{}
router := toolbox.NewServiceRouter(
toolbox.ServiceRouting{
HTTPMethod: "GET",
URI: "/v1/reverse/{ids}",
Handler: service.Reverse,
Parameters: []string{"ids"},
},
toolbox.ServiceRouting{
HTTPMethod: "POST",
URI: "/v1/reverse/",
Handler: service.Reverse,
Parameters: []string{"ids"},
},
toolbox.ServiceRouting{
HTTPMethod: "DELETE",
URI: "/v1/delete/{ids}",
Handler: service.Reverse,
Parameters: []string{"ids"},
},
toolbox.ServiceRouting{
HTTPMethod: "GET",
URI: "/v1/reverse2/{ids}",
Handler: service.Reverse,
Parameters: []string{"ids"},
HandlerInvoker: ReverseInvoker,
},
toolbox.ServiceRouting{
HTTPMethod: "GET",
URI: "/v1/tasks",
Parameters: []string{"status"},
Handler: func(status string) map[string]interface{} {
var result = map[string]interface{}{
"STATUS": status,
"ABc": 101,
}
return result
},
},
toolbox.ServiceRouting{
HTTPMethod: "GET",
URI: "/v1/tasks/{ids}",
Parameters: []string{"ids"},
Handler: func(ids ...string) map[string]interface{} {
var result = map[string]interface{}{
"STATUS": ids,
"ABc": 102,
}
return result
},
},
)
http.HandleFunc("/v1/", func(writer http.ResponseWriter, reader *http.Request) {
err := router.Route(writer, reader)
assert.Nil(t, err)
})
fmt.Printf("Started test server on port %v\n", port)
log.Fatal(http.ListenAndServe(":"+port, nil))
}
func TestServiceRouter(t *testing.T) {
go func() {
StartServer("8082", t)
}()
time.Sleep(2 * time.Second)
{
var result = map[string]interface{}{}
err := toolbox.RouteToService("get", "http://127.0.0.1:8082/v1/tasks/1,2,3", nil, &result)
if err != nil {
t.Errorf("failed to send get request %v", err)
}
assert.EqualValues(t, []interface{}{"1", "2", "3"}, result["STATUS"])
}
//
{
var result = map[string]interface{}{}
err := toolbox.RouteToService("get", "http://127.0.0.1:8082/v1/tasks?status=OK", nil, &result)
if err != nil {
t.Errorf("failed to send get request %v", err)
}
assert.EqualValues(t, "OK", result["STATUS"])
}
var result = make([]int, 0)
{
err := toolbox.RouteToService("get", "http://127.0.0.1:8082/v1/reverse/1,7,3", nil, &result)
if err != nil {
t.Errorf("failed to send get request %v", err)
}
assert.EqualValues(t, []int{3, 7, 1}, result)
}
{
err := toolbox.RouteToService("post", "http://127.0.0.1:8082/v1/reverse/", []int{1, 7, 3}, &result)
if err != nil {
t.Errorf("failed to send get request %v", err)
}
assert.EqualValues(t, []int{3, 7, 1}, result)
}
{
err := toolbox.RouteToService("delete", "http://127.0.0.1:8082/v1/delete/", []int{1, 7, 3}, &result)
if err != nil {
t.Errorf("failed to send delete request %v", err)
}
assert.EqualValues(t, []int{3, 7, 1}, result)
}
{
err := toolbox.RouteToService("delete", "http://127.0.0.1:8082/v1/delete/1,7,3", nil, &result)
if err != nil {
t.Errorf("failed to send delete request %v", err)
}
assert.EqualValues(t, []int{3, 7, 1}, result)
}
{ //Test custom handler invocation without reflection
err := toolbox.RouteToService("get", "http://127.0.0.1:8082/v1/reverse2/1,7,3", nil, &result)
if err != nil {
t.Errorf("failed to send delete request %v", err)
}
assert.EqualValues(t, []int{3, 7, 1}, result)
}
}
toolbox-0.33.2/ssh/ 0000775 0000000 0000000 00000000000 13741102511 0014055 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/ssh/replay_command.go 0000664 0000000 0000000 00000010556 13741102511 0017405 0 ustar 00root root 0000000 0000000 package ssh
import (
"fmt"
"github.com/viant/toolbox"
"io/ioutil"
"os"
"path"
"sort"
"strings"
)
//ReplayCommand represent a replay command
type ReplayCommand struct {
Stdin string
Index int
Stdout []string
Error string
}
//replayCommands represnets command grouped by stdin
type ReplayCommands struct {
Commands map[string]*ReplayCommand
Keys []string
BaseDir string
}
//Register register stdin and corresponding stdout conversation
func (c *ReplayCommands) Register(stdin, stdout string) {
if _, has := c.Commands[stdin]; !has {
c.Commands[stdin] = &ReplayCommand{
Stdin: stdin,
Stdout: make([]string, 0),
}
c.Keys = append(c.Keys, stdin)
}
c.Commands[stdin].Stdout = append(c.Commands[stdin].Stdout, stdout)
}
//return stdout pointed by index and increases index or empty string if exhausted
func (c *ReplayCommands) Next(stdin string) string {
var stdout = c.Commands[stdin].Stdout
var index = c.Commands[stdin].Index
if index < len(stdout) {
c.Commands[stdin].Index++
return stdout[index]
}
return ""
}
func (c *ReplayCommands) Enable(source interface{}) (err error) {
switch value := source.(type) {
case *service:
value.replayCommands = c
value.recordSession = true
case *multiCommandSession:
value.replayCommands = c
value.recordSession = true
default:
err = fmt.Errorf("unsupported type: %T", source)
}
return err
}
func (c *ReplayCommands) Disable(source interface{}) (err error) {
switch value := source.(type) {
case *service:
value.replayCommands = nil
value.recordSession = false
case *multiCommandSession:
value.replayCommands = nil
value.recordSession = false
default:
err = fmt.Errorf("unsupported type: %T", source)
}
return err
}
//Store stores replay command in the base directory
func (c *ReplayCommands) Store() error {
err := toolbox.CreateDirIfNotExist(c.BaseDir)
if err != nil {
return err
}
for i, key := range c.Keys {
var command = c.Commands[key]
var filenamePrefix = path.Join(c.BaseDir, fmt.Sprintf("%03d", i+1))
var stdinFilename = filenamePrefix + "_000.stdin"
err := ioutil.WriteFile(stdinFilename, []byte(command.Stdin), 0644)
if err != nil {
return err
}
for j, stdout := range command.Stdout {
var stdoutFilename = fmt.Sprintf("%v_%03d.stdout", filenamePrefix, j+1)
err := ioutil.WriteFile(stdoutFilename, []byte(stdout), 0644)
if err != nil {
return err
}
}
}
return nil
}
//Load loads replay command from base directory
func (c *ReplayCommands) Load() error {
parent, err := os.Open(c.BaseDir)
if err != nil {
return err
}
files, err := parent.Readdir(1000)
if err != nil {
return err
}
var stdinMap = make(map[string]string)
var stdoutMap = make(map[string]string)
for _, candidate := range files {
ext := path.Ext(candidate.Name())
var contentMap map[string]string
if ext == ".stdin" {
contentMap = stdinMap
} else if ext == ".stdout" {
contentMap = stdoutMap
} else {
continue
}
var filename = path.Join(c.BaseDir, candidate.Name())
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil
}
contentMap[candidate.Name()] = string(content)
}
for key, stdin := range stdinMap {
var prefix = key[:len(key)-10]
var candidateKeys = toolbox.MapKeysToStringSlice(stdoutMap)
sort.Strings(candidateKeys)
for _, candidateKey := range candidateKeys {
if strings.HasPrefix(candidateKey, prefix) {
stdout := stdoutMap[candidateKey]
c.Register(stdin, stdout)
}
}
}
return nil
}
//Shell returns command shell
func (c *ReplayCommands) Shell() string {
for _, candidate := range c.Commands {
if strings.HasPrefix(candidate.Stdin, "PS1=") && len(candidate.Stdout) > 0 {
return candidate.Stdout[0]
}
}
return ""
}
//System returns system name
func (c *ReplayCommands) System() string {
for _, candidate := range c.Commands {
if strings.HasPrefix(candidate.Stdin, "uname -s") && len(candidate.Stdout) > 0 {
return strings.ToLower(candidate.Stdout[0])
}
}
return ""
}
//NewReplayCommands create a new replay commands or error if provided basedir does not exists and can not be created
func NewReplayCommands(basedir string) (*ReplayCommands, error) {
if !toolbox.FileExists(basedir) {
err := os.MkdirAll(basedir, 0744)
if err != nil {
return nil, err
}
}
return &ReplayCommands{
Commands: make(map[string]*ReplayCommand),
Keys: make([]string, 0),
BaseDir: basedir,
}, nil
}
toolbox-0.33.2/ssh/service.go 0000664 0000000 0000000 00000015544 13741102511 0016055 0 ustar 00root root 0000000 0000000 package ssh
import (
"bytes"
"fmt"
"github.com/pkg/errors"
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/storage"
"golang.org/x/crypto/ssh"
"io"
"net"
"os"
"path"
"strings"
"sync"
"time"
)
type (
//Service represents ssh service
Service interface {
//Service returns a service wrapper
Client() *ssh.Client
//OpenMultiCommandSession opens multi command session
OpenMultiCommandSession(config *SessionConfig) (MultiCommandSession, error)
//Run runs supplied command
Run(command string) error
//Upload uploads provided content to specified destination
//Deprecated: please consider using https://github.com/viant/afs/tree/master/scp
Upload(destination string, mode os.FileMode, content []byte) error
//Download downloads content from specified source.
//Deprecated: please consider using https://github.com/viant/afs/tree/master/scp
Download(source string) ([]byte, error)
//OpenTunnel opens a tunnel between local to remote for network traffic.
OpenTunnel(localAddress, remoteAddress string) error
NewSession() (*ssh.Session, error)
Close() error
}
)
//service represnt SSH service
type service struct {
host string
client *ssh.Client
forwarding []*Tunnel
replayCommands *ReplayCommands
recordSession bool
config *ssh.ClientConfig
}
//Service returns undelying ssh Service
func (c *service) Client() *ssh.Client {
return c.client
}
//Service returns undelying ssh Service
func (c *service) NewSession() (*ssh.Session, error) {
return c.client.NewSession()
}
//MultiCommandSession create a new MultiCommandSession
func (c *service) OpenMultiCommandSession(config *SessionConfig) (MultiCommandSession, error) {
return newMultiCommandSession(c, config, c.replayCommands, c.recordSession)
}
func (c *service) Run(command string) error {
session, err := c.client.NewSession()
if err != nil {
panic("failed to create session: " + err.Error())
}
defer session.Close()
return session.Run(command)
}
func (c *service) transferData(payload []byte, createFileCmd string, writer io.Writer, errors chan error, waitGroup *sync.WaitGroup) {
const endSequence = "\x00"
defer waitGroup.Done()
_, err := fmt.Fprint(writer, createFileCmd)
if err != nil {
errors <- err
return
}
_, err = io.Copy(writer, bytes.NewReader(payload))
if err != nil {
errors <- err
return
}
if _, err = fmt.Fprint(writer, endSequence); err != nil {
errors <- err
return
}
}
type Errors chan error
func (e Errors) GetError() error {
select {
case err := <-e:
return err
case <-time.After(time.Millisecond):
}
return nil
}
const operationSuccessful = 0
func checkOutput(reader io.Reader, errorChannel Errors) {
writer := new(bytes.Buffer)
io.Copy(writer, reader)
if writer.Len() > 1 {
data := writer.Bytes()
if data[1] == operationSuccessful {
return
} else if len(data) > 2 {
errorChannel <- errors.New(string(data[2:]))
}
}
}
//Upload uploads passed in content into remote destination
func (c *service) Upload(destination string, mode os.FileMode, content []byte) (err error) {
err = c.upload(destination, mode, content)
if err != nil {
if strings.Contains(err.Error(), "No such file or directory") {
dir, _ := path.Split(destination)
c.Run("mkdir -p " + dir)
return c.upload(destination, mode, content)
} else if strings.Contains(err.Error(), "handshake") || strings.Contains(err.Error(), "connection") {
time.Sleep(500 * time.Millisecond)
fmt.Printf("got error %v\n", err)
c.Reconnect()
return c.upload(destination, mode, content)
}
}
return err
}
func (c *service) getSession() (*ssh.Session, error) {
return c.client.NewSession()
}
//Upload uploads passed in content into remote destination
func (c *service) upload(destination string, mode os.FileMode, content []byte) (err error) {
dir, file := path.Split(destination)
if mode == 0 {
mode = 0644
}
waitGroup := &sync.WaitGroup{}
waitGroup.Add(1)
if strings.HasPrefix(file, "/") {
file = string(file[1:])
}
session, err := c.getSession()
if err != nil {
return err
}
writer, err := session.StdinPipe()
if err != nil {
return errors.Wrap(err, "failed to acquire stdin")
}
defer writer.Close()
var transferError Errors = make(chan error, 1)
defer close(transferError)
var sessionError Errors = make(chan error, 1)
defer close(sessionError)
output, err := session.StdoutPipe()
if err != nil {
return errors.Wrap(err, "failed to acquire stdout")
}
go checkOutput(output, sessionError)
if mode >= 01000 {
mode = storage.DefaultFileMode
}
fileMode := string(fmt.Sprintf("C%04o", mode)[:5])
createFileCmd := fmt.Sprintf("%v %d %s\n", fileMode, len(content), file)
go c.transferData(content, createFileCmd, writer, transferError, waitGroup)
scpCommand := "scp -qtr " + dir
err = session.Start(scpCommand)
if err != nil {
return err
}
waitGroup.Wait()
writerErr := writer.Close()
if err := sessionError.GetError(); err != nil {
return err
}
if err := transferError.GetError(); err != nil {
return err
}
if err = session.Wait(); err != nil {
if err := sessionError.GetError(); err != nil {
return err
}
return err
}
return writerErr
}
//Download download passed source file from remote host.
func (c *service) Download(source string) ([]byte, error) {
session, err := c.client.NewSession()
if err != nil {
return nil, err
}
defer session.Close()
return session.Output(fmt.Sprintf("cat %s", source))
}
//Host returns client host
func (c *service) Host() string {
return c.host
}
//Close closes service
func (c *service) Close() error {
if len(c.forwarding) > 0 {
for _, forwarding := range c.forwarding {
_ = forwarding.Close()
}
}
return c.client.Close()
}
//Reconnect client
func (c *service) Reconnect() error {
return c.connect()
}
//OpenTunnel tunnels data between localAddress and remoteAddress on ssh connection
func (c *service) OpenTunnel(localAddress, remoteAddress string) error {
local, err := net.Listen("tcp", localAddress)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to listen on local: %v %v", localAddress))
}
var forwarding = NewForwarding(c.client, remoteAddress, local)
if len(c.forwarding) == 0 {
c.forwarding = make([]*Tunnel, 0)
}
c.forwarding = append(c.forwarding, forwarding)
go forwarding.Handle()
return nil
}
func (c *service) connect() (err error) {
if c.client, err = ssh.Dial("tcp", c.host, c.config); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to dial %v: %s", c.host))
}
return nil
}
//NewService create a new ssh service, it takes host port and authentication config
func NewService(host string, port int, authConfig *cred.Config) (Service, error) {
if authConfig == nil {
authConfig = &cred.Config{}
}
clientConfig, err := authConfig.ClientConfig()
if err != nil {
return nil, err
}
var result = &service{
host: fmt.Sprintf("%s:%d", host, port),
config: clientConfig,
}
return result, result.connect()
}
toolbox-0.33.2/ssh/service_replay.go 0000664 0000000 0000000 00000003677 13741102511 0017435 0 ustar 00root root 0000000 0000000 package ssh
import (
"errors"
"fmt"
"golang.org/x/crypto/ssh"
"os"
)
//Service represents ssh service
type replayService struct {
storage map[string][]byte
shellPrompt string
system string
commands *ReplayCommands
}
//Service returns a service wrapper
func (s *replayService) Client() *ssh.Client {
return &ssh.Client{}
}
//OpenMultiCommandSession opens multi command session
func (s *replayService) OpenMultiCommandSession(config *SessionConfig) (MultiCommandSession, error) {
return NewReplayMultiCommandSession(s.shellPrompt, s.system, s.commands), nil
}
//Run runs supplied command
func (s *replayService) Run(command string) error {
if commands, ok := s.commands.Commands[command]; ok {
if commands.Error != "" {
return errors.New(commands.Error)
}
}
s.commands.Next(command)
return nil
}
//Upload uploads provided content to specified destination
func (s *replayService) Upload(destination string, mode os.FileMode, content []byte) error {
s.storage[destination] = content
return nil
}
//Download downloads content from specified source.
func (s *replayService) Download(source string) ([]byte, error) {
if _, has := s.storage[source]; !has {
return nil, fmt.Errorf("no such file or directory")
}
return s.storage[source], nil
}
//OpenTunnel opens a tunnel between local to remote for network traffic.
func (s *replayService) OpenTunnel(localAddress, remoteAddress string) error {
return nil
}
func (s *replayService) NewSession() (*ssh.Session, error) {
return &ssh.Session{}, nil
}
func (s *replayService) Reconnect() error {
return errors.New("unsupported")
}
func (s *replayService) Close() error {
return nil
}
func NewReplayService(shellPrompt, system string, commands *ReplayCommands, storage map[string][]byte) Service {
if len(storage) == 0 {
storage = make(map[string][]byte)
}
return &replayService{
storage: storage,
shellPrompt: shellPrompt,
system: system,
commands: commands,
}
}
toolbox-0.33.2/ssh/service_replay_test.go 0000664 0000000 0000000 00000001605 13741102511 0020461 0 ustar 00root root 0000000 0000000 package ssh_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/ssh"
"path"
"testing"
)
func Test_NewReplayService(t *testing.T) {
parent := toolbox.CallerDirectory(3)
commands, err := ssh.NewReplayCommands(path.Join(parent, "test/ls"))
assert.Nil(t, err)
err = commands.Load()
if !assert.Nil(t, err) {
return
}
assert.Equal(t, 3, len(commands.Commands))
assert.Nil(t, err)
defer commands.Store()
service := ssh.NewReplayService("AWITAS-C02TF066H040:awitas1511796457759720702$", "darwin", commands, nil)
if err == nil {
assert.NotNil(t, service)
session, err := service.OpenMultiCommandSession(nil)
assert.Nil(t, err)
defer session.Close()
assert.NotNil(t, session)
var out string
out, err = session.Run("ls /etc/hosts", nil, 2000)
assert.Equal(t, "/etc/hosts", out)
} else {
assert.Nil(t, service)
}
}
toolbox-0.33.2/ssh/service_test.go 0000664 0000000 0000000 00000004364 13741102511 0017112 0 ustar 00root root 0000000 0000000 package ssh_test
import (
"fmt"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/ssh"
"io/ioutil"
"os"
"path"
"testing"
)
func TestNewClient(t *testing.T) {
commands, err := ssh.NewReplayCommands("/tmp/ls")
if err != nil {
return
}
defer commands.Store()
service, err := ssh.NewService("127.0.0.1", 22, nil)
if err != nil {
return
}
assert.NotNil(t, service)
commands.Enable(service)
if err == nil {
assert.NotNil(t, service)
session, err := service.OpenMultiCommandSession(nil)
assert.Nil(t, err)
defer session.Close()
assert.NotNil(t, session)
var out string
out, err = session.Run("ls /etc/hosts", nil, 2000)
assert.Equal(t, "/etc/hosts", out)
} else {
assert.Nil(t, service)
}
}
func TestClient_Upload(t *testing.T) {
service, err := ssh.NewService("127.0.0.1", 22, nil)
if err != nil {
return
}
if err == nil {
assert.NotNil(t, service)
err = service.Upload("/tmp/a/abcd.bin", 0644, []byte{0x1, 0x6, 0x10})
assert.Nil(t, err)
content, err := service.Download("/tmp/a/abcd.bin")
assert.Nil(t, err)
assert.Equal(t, []byte{0x1, 0x6, 0x10}, (content))
} else {
assert.Nil(t, service)
}
}
func TestClient_Key(t *testing.T) {
auth, err := cred.NewConfig("/Users/awitas/.secret/scp1.json")
if err != nil {
return
}
assert.Nil(t, err)
service, err := ssh.NewService("127.0.0.1", 22, auth)
if err != nil {
return
}
assert.NotNil(t, service)
}
func TestClient_UploadLargeFile(t *testing.T) {
service, err := ssh.NewService("127.0.0.1", 22, nil)
if err != nil {
return
}
tempdir := os.TempDir()
filename := path.Join(tempdir, "kkk/.bin/largefile.bin")
toolbox.RemoveFileIfExist(filename)
//defer toolbox.RemoveFileIfExist(filename)
//file, err := os.OpenFile(filename, os.O_RDWR | os.O_CREATE, 0644)
//assert.Nil(t, err)
var payload = make([]byte, 1024*1024*16)
for i := 0; i < len(payload); i += 32 {
payload[i] = byte(i % 256)
}
//file.Write(payload)
//file.Close()
err = service.Upload(filename, 0644, payload)
fmt.Printf("%v\n", err)
assert.Nil(t, err)
data, err := ioutil.ReadFile(filename)
assert.Nil(t, err)
if assert.Equal(t, len(payload), len(data)) {
assert.EqualValues(t, data, payload)
}
}
toolbox-0.33.2/ssh/session.go 0000664 0000000 0000000 00000027154 13741102511 0016100 0 ustar 00root root 0000000 0000000 package ssh
import (
"bytes"
"fmt"
"github.com/lunixbochs/vtclean"
"github.com/pkg/errors"
"github.com/viant/toolbox"
"golang.org/x/crypto/ssh"
"io"
"path"
"strings"
"sync"
"sync/atomic"
"time"
)
type TerminatedError struct {
Err error
}
func (t *TerminatedError) Error() string {
return fmt.Sprintf("terminated due to %v", t.Err)
}
//ErrTerminated - command session terminated
var ErrTerminated = &TerminatedError{}
const defaultShell = "/bin/bash"
const (
drainTimeoutMs = 10
defaultTimeoutMs = 20000
stdoutFlashFrequencyMs = 1000
initTimeoutMs = 300
defaultTickFrequency = 100
)
//Listener represent command listener (it will send stdout fragments as thier being available on stdout)
type Listener func(stdout string, hasMore bool)
//MultiCommandSession represents a multi command session
type MultiCommandSession interface {
Run(command string, listener Listener, timeoutMs int, terminators ...string) (string, error)
ShellPrompt() string
System() string
Reconnect() error
Close()
}
//multiCommandSession represents a multi command session
//a new command are send vi stdin
type multiCommandSession struct {
service *service
config *SessionConfig
replayCommands *ReplayCommands
recordSession bool
session *ssh.Session
stdOutput chan string
stdError chan string
stdInput io.WriteCloser
promptSequence string
shellPrompt string
escapedShellPrompt string
system string
running int32
stdin string
}
func (s *multiCommandSession) Run(command string, listener Listener, timeoutMs int, terminators ...string) (string, error) {
if atomic.LoadInt32(&s.running) == 0 {
return "", ErrTerminated
}
s.drainStdout()
if !strings.HasSuffix(command, "\n") {
command += "\n"
}
var stdin = command
s.stdin = stdin
_, err := s.stdInput.Write([]byte(stdin))
if err != nil {
return "", fmt.Errorf("failed to execute command: %v, err: %v", command, err)
}
var output string
output, _, err = s.readResponse(timeoutMs, listener, terminators...)
if s.recordSession {
s.replayCommands.Register(stdin, output)
}
return output, err
}
//ShellPrompt returns a shell prompt
func (s *multiCommandSession) ShellPrompt() string {
return s.shellPrompt
}
//System returns a system name
func (s *multiCommandSession) System() string {
return s.system
}
//Close closes the session with its resources
func (s *multiCommandSession) Close() {
atomic.StoreInt32(&s.running, 0)
_ = s.stdInput.Close()
if s.session != nil {
_ = s.session.Close()
}
}
func (s *multiCommandSession) closeIfError(err error) bool {
if err != nil {
if err != ErrTerminated {
ErrTerminated.Err = err
}
s.Close()
return true
}
return false
}
func (s *multiCommandSession) start(shell string) (output string, err error) {
var reader, errReader io.Reader
reader, err = s.session.StdoutPipe()
if err != nil {
return "", err
}
errReader, err = s.session.StderrPipe()
if err != nil {
return "", err
}
waitGroup := sync.WaitGroup{}
waitGroup.Add(2)
go func() {
waitGroup.Done()
s.copy(reader, s.stdOutput)
}()
go func() {
waitGroup.Done()
s.copy(errReader, s.stdError)
}()
if shell == "" {
shell = defaultShell
}
waitGroup.Wait()
s.stdin = shell
err = s.session.Start(shell)
if err != nil {
return "", err
}
_, name := path.Split(shell)
output, _, err = s.readResponse(defaultTimeoutMs, nil, name)
return output, err
}
//copy copy data from reader to channel
func (s *multiCommandSession) copy(reader io.Reader, out chan string) {
var written int64 = 0
buf := make([]byte, 128*1024)
var err error
var bytesRead int
for {
writer := new(bytes.Buffer)
if atomic.LoadInt32(&s.running) == 0 {
return
}
bytesRead, err = reader.Read(buf)
if bytesRead > 0 {
bytesWritten, writeError := writer.Write(buf[:bytesRead])
if s.closeIfError(writeError) {
return
}
if bytesWritten > 0 {
written += int64(bytesWritten)
}
if bytesRead != bytesWritten {
if s.closeIfError(io.ErrShortWrite) {
return
}
}
out <- string(writer.Bytes())
}
if s.closeIfError(err) {
return
}
}
}
func escapeInput(input string) string {
input = vtclean.Clean(input, false)
if input == "" {
return input
}
return strings.Trim(input, "\n\r\t ")
}
func (s *multiCommandSession) Reconnect() (err error) {
atomic.StoreInt32(&s.running, 1)
s.service.Reconnect()
s.session, err = s.service.client.NewSession()
defer func() {
if err != nil {
s.service.client.Close()
}
}()
if err != nil {
return err
}
return s.init()
}
func (s *multiCommandSession) hasPrompt(input string) bool {
escapedInput := escapeInput(input)
var shellPrompt = s.shellPrompt
if shellPrompt == "" {
shellPrompt = "$"
}
if s.escapedShellPrompt == "" && s.shellPrompt != "" {
s.escapedShellPrompt = escapeInput(s.shellPrompt)
}
if s.escapedShellPrompt != "" && strings.HasSuffix(escapedInput, s.escapedShellPrompt) || strings.HasSuffix(input, s.shellPrompt) {
return true
}
return false
}
func (s *multiCommandSession) hasTerminator(input string, terminators ...string) bool {
escapedInput := escapeInput(input)
input = escapedInput
for _, candidate := range terminators {
candidateLen := len(candidate)
if candidateLen == 0 {
continue
}
if candidate[0:1] == "^" && strings.HasPrefix(input, candidate[1:]) {
return true
}
if candidate[candidateLen-1:] == "$" && strings.HasSuffix(input, candidate[:candidateLen-1]) {
return true
}
if strings.Contains(input, candidate) {
return true
}
}
return false
}
func (s *multiCommandSession) removePromptIfNeeded(stdout string) string {
if strings.Contains(stdout, s.shellPrompt) {
stdout = strings.Replace(stdout, s.shellPrompt, "", 1)
var lines = []string{}
for _, line := range strings.Split(stdout, "\n") {
if strings.TrimSpace(line) == "" {
continue
}
lines = append(lines, line)
}
stdout = strings.Join(lines, "\n")
}
return stdout
}
func (s *multiCommandSession) readResponse(timeoutMs int, listener Listener, terminators ...string) (out string, has bool, err error) {
var hasPrompt, hasTerminator bool
if timeoutMs == 0 {
timeoutMs = defaultTimeoutMs
}
notification := newNotificationWindow(listener, stdoutFlashFrequencyMs)
defer notification.flush()
var done int32
defer atomic.StoreInt32(&done, 1)
var errOut string
var hasOutput bool
var waitTimeMs = 0
var tickFrequencyMs = defaultTickFrequency
if tickFrequencyMs > timeoutMs {
tickFrequencyMs = timeoutMs
}
var timeoutDuration = time.Duration(tickFrequencyMs) * time.Millisecond
outer:
for {
select {
case partialOutput := <-s.stdOutput:
waitTimeMs = 0
out += partialOutput
hasTerminator = s.hasTerminator(out, terminators...)
if len(partialOutput) > 0 {
if hasTerminator {
partialOutput = addLineBreakIfNeeded(partialOutput)
}
notification.notify(s.removePromptIfNeeded(partialOutput))
}
hasPrompt = s.hasPrompt(out)
if (hasPrompt || hasTerminator) && len(s.stdOutput) == 0 {
break outer
}
case e := <-s.stdError:
errOut += e
notification.notify(s.removePromptIfNeeded(e))
hasPrompt = s.hasPrompt(errOut)
hasTerminator = s.hasTerminator(errOut, terminators...)
if (hasPrompt || hasTerminator) && len(s.stdOutput) == 0 {
break outer
}
case <-time.After(timeoutDuration):
waitTimeMs += tickFrequencyMs
if waitTimeMs >= timeoutMs {
break outer
}
}
}
if hasTerminator {
s.drainStdout()
}
if errOut != "" {
err = errors.New(errOut)
}
if len(out) > 0 {
hasOutput = true
out = s.removePromptIfNeeded(out)
}
return out, hasOutput, err
}
func addLineBreakIfNeeded(text string) string {
index := strings.LastIndex(text, "\n")
if index == -1 {
return text + "\n"
}
lastFragment := string(text[index:])
if strings.TrimSpace(lastFragment) != "" {
return text + "\n"
}
return text
}
func (s *multiCommandSession) drainStdout() {
//read any outstanding output
for {
_, has, _ := s.readResponse(drainTimeoutMs, nil, "")
if !has {
return
}
}
}
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
c := make(chan bool)
go func() {
defer close(c)
wg.Wait()
c <- true
}()
select {
case <-c:
return false // completed normally
case <-time.After(timeout):
return true // timed out
}
}
func (s *multiCommandSession) shellInit() (err error) {
if s.promptSequence != "" {
if _, err = s.Run(s.promptSequence, nil, initTimeoutMs); err != nil {
return err
}
}
var ts = toolbox.AsString(time.Now().UnixNano())
var waitGroup = &sync.WaitGroup{}
waitGroup.Add(1)
if s.config.Shell == defaultShell {
s.promptSequence = "PS1=\"" + ts + "\\$\""
s.shellPrompt = ts + "$"
s.escapedShellPrompt = escapeInput(s.shellPrompt)
}
var listener Listener
listener = func(stdout string, hasMore bool) {
if !hasMore {
waitGroup.Done()
}
}
_, err = s.Run("", listener, initTimeoutMs, "$")
waitTimeout(waitGroup, 60*time.Second)
s.drainStdout()
_, err = s.Run(s.promptSequence, nil, defaultTimeoutMs, "$")
if s.closeIfError(err) {
return err
}
for i := 0; i < 3; i++ {
s.system, err = s.Run("uname -s", nil, initTimeoutMs)
s.system = strings.ToLower(strings.TrimSpace(s.system))
if strings.TrimSpace(s.system) != "" && !strings.Contains(s.system, "$") {
break
}
}
s.drainStdout()
return nil
}
func (s *multiCommandSession) init() (err error) {
s.session, err = s.service.client.NewSession()
defer func() {
if err != nil {
s.service.client.Close()
}
}()
s.stdOutput = make(chan string)
s.stdError = make(chan string)
for k, v := range s.config.EnvVariables {
err = s.session.Setenv(k, v)
if err != nil {
return err
}
}
modes := ssh.TerminalModes{
ssh.ECHO: 0, // disable echoing
ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
}
if err := s.session.RequestPty(s.config.Term, s.config.Rows, s.config.Columns, modes); err != nil {
return err
}
if s.stdInput, err = s.session.StdinPipe(); err != nil {
return err
}
stdout, err := s.start(s.config.Shell)
if s.closeIfError(err) {
return err
}
if err = checkNotFound(stdout); err != nil {
return fmt.Errorf("failed to open %v shell, %v", s.config.Shell, err)
}
return s.shellInit()
}
func checkNotFound(output string) error {
if strings.Contains(output, "not found") {
return fmt.Errorf("failed run %s", output)
}
return nil
}
func newMultiCommandSession(service *service, config *SessionConfig, replayCommands *ReplayCommands, recordSession bool) (MultiCommandSession, error) {
if config == nil {
config = &SessionConfig{}
}
config.applyDefault()
result := &multiCommandSession{
service: service,
config: config,
running: 1,
recordSession: recordSession,
replayCommands: replayCommands,
}
return result, result.init()
}
type notificationWindow struct {
checkpoint *time.Time
listener Listener
elapsedMs int
stdout string
frequencyMs int
}
func (t *notificationWindow) flush() {
if t.listener == nil {
return
}
if t.stdout != "" {
t.listener(t.stdout, true)
}
t.listener("", false)
}
func (t *notificationWindow) notify(stdout string) {
var now = time.Now()
if t.listener == nil {
return
}
t.stdout += stdout
t.elapsedMs += int(now.Sub(*t.checkpoint) / time.Millisecond)
t.checkpoint = &now
if t.elapsedMs > t.frequencyMs {
t.listener(t.stdout, true)
t.stdout = ""
t.elapsedMs = 0
}
}
func newNotificationWindow(listener Listener, frequencyMs int) *notificationWindow {
var now = time.Now()
return ¬ificationWindow{
checkpoint: &now,
listener: listener,
frequencyMs: frequencyMs,
}
}
toolbox-0.33.2/ssh/session_config.go 0000664 0000000 0000000 00000000633 13741102511 0017416 0 ustar 00root root 0000000 0000000 package ssh
//SessionConfig represents a new session config
type SessionConfig struct {
EnvVariables map[string]string
Shell string
Term string
Rows int
Columns int
}
func (c *SessionConfig) applyDefault() {
if c.Shell == "" {
c.Shell = "/bin/bash"
}
if c.Term == "" {
c.Term = "xterm"
}
if c.Rows == 0 {
c.Rows = 100
}
if c.Columns == 0 {
c.Columns = 100
}
}
toolbox-0.33.2/ssh/session_replay.go 0000664 0000000 0000000 00000002143 13741102511 0017443 0 ustar 00root root 0000000 0000000 package ssh
import (
"errors"
"strings"
)
const commandNotFound = "Command not found"
type replayMultiCommandSession struct {
shellPrompt string
system string
replay *ReplayCommands
}
func (s *replayMultiCommandSession) Run(command string, listener Listener, timeoutMs int, terminators ...string) (string, error) {
if !strings.HasSuffix(command, "\n") {
command = command + "\n"
}
replay, ok := s.replay.Commands[command]
if !ok {
return commandNotFound, nil
}
if replay.Error != "" {
return "", errors.New(replay.Error)
}
return s.replay.Next(command), nil
}
func (s *replayMultiCommandSession) Reconnect() error {
return errors.New("unsupported")
}
func (s *replayMultiCommandSession) ShellPrompt() string {
return s.shellPrompt
}
func (s *replayMultiCommandSession) System() string {
return s.system
}
func (s *replayMultiCommandSession) Close() {
}
func NewReplayMultiCommandSession(shellPrompt, system string, commands *ReplayCommands) MultiCommandSession {
return &replayMultiCommandSession{
shellPrompt: shellPrompt,
system: system,
replay: commands,
}
}
toolbox-0.33.2/ssh/test/ 0000775 0000000 0000000 00000000000 13741102511 0015034 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/ssh/test/ls/ 0000775 0000000 0000000 00000000000 13741102511 0015452 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/ssh/test/ls/001_000.stdin 0000664 0000000 0000000 00000000011 13741102511 0017364 0 ustar 00root root 0000000 0000000 uname -s
toolbox-0.33.2/ssh/test/ls/001_001.stdout 0000664 0000000 0000000 00000000006 13741102511 0017572 0 ustar 00root root 0000000 0000000 Darwin toolbox-0.33.2/ssh/test/ls/002_000.stdin 0000664 0000000 0000000 00000000016 13741102511 0017372 0 ustar 00root root 0000000 0000000 ls /etc/hosts
toolbox-0.33.2/ssh/test/ls/002_001.stdout 0000664 0000000 0000000 00000000012 13741102511 0017570 0 ustar 00root root 0000000 0000000 /etc/hosts toolbox-0.33.2/ssh/test/ls/003_000.stdin 0000664 0000000 0000000 00000000041 13741102511 0017371 0 ustar 00root root 0000000 0000000 PS1="\h:\u1511796457759720702\$"
toolbox-0.33.2/ssh/test/ls/003_001.stdout 0000664 0000000 0000000 00000000056 13741102511 0017601 0 ustar 00root root 0000000 0000000 AWITAS-C02TF066H040:awitas1511796457759720702$ toolbox-0.33.2/ssh/tunnel.go 0000664 0000000 0000000 00000003527 13741102511 0015720 0 ustar 00root root 0000000 0000000 package ssh
import (
"fmt"
"golang.org/x/crypto/ssh"
"io"
"log"
"net"
"sync"
"sync/atomic"
)
//Tunnel represents a SSH forwarding link
type Tunnel struct {
RemoteAddress string
client *ssh.Client
Local net.Listener
Connections []net.Conn
mutex *sync.Mutex
closed int32
}
func (f *Tunnel) tunnelTraffic(local, remote net.Conn) {
defer local.Close()
defer remote.Close()
completionChannel := make(chan bool)
go func() {
_, err := io.Copy(local, remote)
if err != nil {
log.Printf("failed to copy remote to local: %v", err)
}
completionChannel <- true
}()
go func() {
_, _ = io.Copy(remote, local)
//if err != nil {
// log.Printf("failed to copy local to remote: %v", err)
//}
completionChannel <- true
}()
<-completionChannel
}
//Handle listen on local client to create tunnel with remote address.
func (f *Tunnel) Handle() error {
for {
if atomic.LoadInt32(&f.closed) == 1 {
return nil
}
localclient, err := f.Local.Accept()
if err != nil {
return err
}
remote, err := f.client.Dial("tcp", f.RemoteAddress)
if err != nil {
return fmt.Errorf("failed to connect to remote: %v %v", f.RemoteAddress, err)
}
f.Connections = append(f.Connections, remote)
f.Connections = append(f.Connections, localclient)
go f.tunnelTraffic(localclient, remote)
}
return nil
}
//Close closes forwarding link
func (f *Tunnel) Close() error {
atomic.StoreInt32(&f.closed, 1)
_ = f.Local.Close()
for _, remote := range f.Connections {
_ = remote.Close()
}
return nil
}
//NewForwarding creates a new ssh forwarding link
func NewForwarding(client *ssh.Client, remoteAddress string, local net.Listener) *Tunnel {
return &Tunnel{
client: client,
RemoteAddress: remoteAddress,
Connections: make([]net.Conn, 0),
Local: local,
mutex: &sync.Mutex{},
}
}
toolbox-0.33.2/stack_helper.go 0000664 0000000 0000000 00000003046 13741102511 0016256 0 ustar 00root root 0000000 0000000 package toolbox
import (
"path"
"runtime"
"strings"
)
// CallerInfo return filename, function or file line from the stack
func CallerInfo(callerIndex int) (string, string, int) {
var callerPointer = make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(callerIndex, callerPointer)
callerInfo := runtime.FuncForPC(callerPointer[0])
file, line := callerInfo.FileLine(callerPointer[0])
callerName := callerInfo.Name()
dotPosition := strings.LastIndex(callerName, ".")
return file, callerName[dotPosition+1:], line
}
//CallerDirectory returns directory of caller source code directory
func CallerDirectory(callerIndex int) string {
file, _, _ := CallerInfo(callerIndex)
parent, _ := path.Split(file)
return parent
}
func hasMatch(target string, candidates ...string) bool {
for _, candidate := range candidates {
if strings.HasSuffix(target, candidate) {
return true
}
}
return false
}
//DiscoverCaller returns the first matched caller info
func DiscoverCaller(offset, maxDepth int, ignoreFiles ...string) (string, string, int) {
var callerPointer = make([]uintptr, maxDepth) // at least 1 entry needed
var caller *runtime.Func
var filename string
var line int
for i := offset; i < maxDepth; i++ {
runtime.Callers(i, callerPointer)
caller = runtime.FuncForPC(callerPointer[0])
filename, line = caller.FileLine(callerPointer[0])
if hasMatch(filename, ignoreFiles...) {
continue
}
break
}
callerName := caller.Name()
dotPosition := strings.LastIndex(callerName, ".")
return filename, callerName[dotPosition+1:], line
}
toolbox-0.33.2/storage/ 0000775 0000000 0000000 00000000000 13741102511 0014724 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/README.md 0000664 0000000 0000000 00000004526 13741102511 0016212 0 ustar 00root root 0000000 0000000 ## Storage API
Deprecated - please use https://github.com/viant/afs API instead
This API provides unified way of accessing any storage system.
It comes with the following implementation so far:
0 {
result = string(result[pathRoot:])
}
if strings.HasSuffix(result, "/") {
result = string(result[:len(result)-1])
}
return result
}
func truncatePath(path string) string {
if len(path) <= 1 {
return path
}
if strings.HasSuffix(path, "/") {
return string(path[:len(path)-1])
}
return path
}
func copyStorageContent(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
sourceListURL := sourceURL
if subPath != "" {
sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
}
objects, err := sourceService.List(sourceListURL)
if err != nil {
return err
}
for _, object := range objects {
if err = copyObject(object, sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, subPath, copyHandler); err != nil {
return err
}
}
return nil
}
func copyObject(object Object, sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
var objectRelativePath string
sourceURLPath := urlPath(sourceURL)
var objectURLPath = urlPath(object.URL())
if object.IsFolder() {
if truncatePath(sourceURLPath) == truncatePath(objectURLPath) {
return nil
}
if subPath != "" && objectURLPath == toolbox.URLPathJoin(sourceURLPath, subPath) {
return nil
}
}
if len(objectURLPath) > len(sourceURLPath) {
objectRelativePath = objectURLPath[len(sourceURLPath):]
if strings.HasPrefix(objectRelativePath, "/") {
objectRelativePath = string(objectRelativePath[1:])
}
}
var destinationObjectURL = destinationURL
if objectRelativePath != "" {
destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
}
if object.IsContent() {
reader, err := sourceService.Download(object)
if err != nil {
err = fmt.Errorf("unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
return err
}
defer reader.Close()
if modifyContentHandler != nil {
content, err := ioutil.ReadAll(reader)
if err != nil {
return err
}
reader = ioutil.NopCloser(bytes.NewReader(content))
reader, err = modifyContentHandler(reader)
if err != nil {
err = fmt.Errorf("unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
return err
}
}
if subPath == "" {
_, sourceName := path.Split(object.URL())
_, destinationName := path.Split(destinationURL)
if strings.HasSuffix(destinationObjectURL, "/") {
destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
} else {
destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
if destinationObject != nil && destinationObject.IsFolder() {
destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
} else if destinationName != sourceName {
if !strings.Contains(destinationName, ".") {
destinationObjectURL = toolbox.URLPathJoin(destinationURL, sourceName)
}
}
}
}
err = copyHandler(object, reader, destinationService, destinationObjectURL)
if err != nil {
return err
}
} else {
if err := copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler); err != nil {
return err
}
}
return nil
}
func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
mode := DefaultFileMode
if fileInfo := sourceObject.FileInfo(); fileInfo != nil {
mode = fileInfo.Mode()
}
err := destinationService.UploadWithMode(destinationURL, mode, reader)
if err != nil {
err = fmt.Errorf("unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
}
return err
}
func getArchiveCopyHandler(archive *zip.Writer, parentURL string) CopyHandler {
return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
var _, relativePath = toolbox.URLSplit(destinationURL)
if destinationURL != parentURL {
relativePath = strings.Replace(destinationURL, parentURL, "", 1)
}
header, err := zip.FileInfoHeader(sourceObject.FileInfo())
if err != nil {
return err
}
header.Method = zip.Deflate
header.Name = relativePath
writer, err := archive.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(writer, reader)
return err
}
}
//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
if copyHandler == nil {
copyHandler = copySourceToDestination
}
if strings.HasSuffix(sourceURL, "//") {
sourceURL = string(sourceURL[:len(sourceURL)-1])
}
err = copyStorageContent(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
if err != nil {
err = fmt.Errorf("failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
}
return err
}
//Archive archives supplied URL assets into zip writer
func Archive(service Service, URL string, writer *zip.Writer) error {
memService := NewMemoryService()
var destURL = "mem:///dev/nul"
return Copy(service, URL, memService, destURL, nil, getArchiveCopyHandler(writer, destURL))
}
func getArchiveCopyHandlerWithFilter(archive *zip.Writer, parentURL string, predicate func(candidate Object) bool) CopyHandler {
return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
if !predicate(sourceObject) {
return nil
}
var _, relativePath = toolbox.URLSplit(destinationURL)
if destinationURL != parentURL {
relativePath = strings.Replace(destinationURL, parentURL, "", 1)
}
header, err := zip.FileInfoHeader(sourceObject.FileInfo())
if err != nil {
return err
}
header.Method = zip.Store
if strings.HasPrefix(relativePath, "/") {
relativePath = string(relativePath[1:])
}
header.Name = relativePath
writer, err := archive.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(writer, reader)
return err
}
}
//Archive archives supplied URL assets into zip writer with supplied filter
func ArchiveWithFilter(service Service, URL string, writer *zip.Writer, predicate func(candidate Object) bool) error {
memService := NewMemoryService()
var destURL = "mem:///dev/nul"
return Copy(service, URL, memService, destURL, nil, getArchiveCopyHandlerWithFilter(writer, destURL, predicate))
}
func getTarCopyHandler(archive *tar.Writer, destParentURL, parentURL string, dirs map[string]bool) CopyHandler {
if strings.HasSuffix(parentURL, "/") {
parentURL = string(parentURL[:len(parentURL)-2])
}
_, root := path.Split(destParentURL)
if root == "." {
root = ""
}
return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
var _, relativePath = toolbox.URLSplit(destinationURL)
if destinationURL != parentURL {
relativePath = strings.Replace(destinationURL, parentURL, "", 1)
}
if strings.HasPrefix(relativePath, "/") {
relativePath = string(relativePath[1:])
}
relativePath = path.Join(root, relativePath)
parent, _ := path.Split(relativePath)
if parent != "" && !dirs[parent] {
tarHeader := &tar.Header{
Name: parent,
Size: int64(0),
Mode: int64(sourceObject.FileInfo().Mode()),
ModTime: sourceObject.FileInfo().ModTime(),
}
if err := archive.WriteHeader(tarHeader); err != nil {
return fmt.Errorf(" unable to write tar header, %v", err)
}
dirs[parent] = true
}
contents := new(bytes.Buffer)
if _, err := io.Copy(contents, reader); err != nil {
return err
}
data := contents.Bytes()
tarHeader := &tar.Header{
Name: relativePath,
Size: int64(len(data)),
Mode: int64(sourceObject.FileInfo().Mode()),
ModTime: sourceObject.FileInfo().ModTime(),
}
if err := archive.WriteHeader(tarHeader); err != nil {
return fmt.Errorf(" unable to write tar header, %v", err)
}
if _, err := archive.Write(data); err != nil {
return fmt.Errorf(" unable to write tar content, %v", err)
}
return nil
}
}
//Tar tar archives supplied URL assets into zip writer
func Tar(service Service, URL string, writer *tar.Writer, includeOwnerDir bool) error {
memService := NewMemoryService()
var destURL = "mem:///dev/nul"
var dirs = make(map[string]bool)
ownerDir := ""
if includeOwnerDir {
ownerDir = URL
}
return Copy(service, URL, memService, destURL, nil, getTarCopyHandler(writer, ownerDir, destURL, dirs))
}
toolbox-0.33.2/storage/copy_test.go 0000664 0000000 0000000 00000003220 13741102511 0017261 0 ustar 00root root 0000000 0000000 package storage_test
import (
"archive/zip"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
_ "github.com/viant/toolbox/storage/scp"
"os"
"path"
"strings"
"testing"
)
func TestCopy(t *testing.T) {
service := storage.NewService()
assert.NotNil(t, service)
parent := toolbox.CallerDirectory(3)
baseUrl := "file://" + parent + "/test"
toolbox.CreateDirIfNotExist(path.Join(parent, "test/target"))
{
sourceURL := path.Join(baseUrl, "source/")
targetURL := path.Join(baseUrl, "target/")
err := storage.Copy(service, sourceURL, service, targetURL, nil, nil)
assert.Nil(t, err)
expectedFiles := []string{
path.Join(parent, "test/target/file1.txt"),
path.Join(parent, "test/target/file2.txt"),
path.Join(parent, "test/target/dir/file.json"),
path.Join(parent, "test/target/dir2/subdir/file1.txt"),
}
for _, file := range expectedFiles {
assert.True(t, toolbox.FileExists(file))
//os.Remove(file)
}
}
}
func TestArchive(t *testing.T) {
memService := storage.NewMemoryService()
memService.Upload("mem://test/copy/archive/file1.txt", strings.NewReader("abc"))
memService.Upload("mem://test/copy/archive/file2.txt", strings.NewReader("xyz"))
memService.Upload("mem://test/copy/archive/config/test.prop", strings.NewReader("123"))
toolbox.RemoveFileIfExist("/tmp/testCopy.zip")
var writer, err = os.OpenFile("/tmp/testCopy.zip", os.O_CREATE|os.O_WRONLY, 06444)
if assert.Nil(t, err) {
defer writer.Close()
archive := zip.NewWriter(writer)
err = storage.Archive(memService, "mem://test/copy/archive/", archive)
assert.Nil(t, err)
archive.Flush()
archive.Close()
}
}
toolbox-0.33.2/storage/file_info.go 0000664 0000000 0000000 00000002450 13741102511 0017206 0 ustar 00root root 0000000 0000000 package storage
import (
"fmt"
"os"
"strings"
"time"
)
type fileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
func (i *fileInfo) Name() string {
return i.name
}
func (i *fileInfo) Size() int64 {
return i.size
}
func (i *fileInfo) Mode() os.FileMode {
return i.mode
}
func (i *fileInfo) ModTime() time.Time {
return i.modTime
}
func (i *fileInfo) IsDir() bool {
return i.isDir
}
func (i *fileInfo) Sys() interface{} {
return i
}
func NewFileInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, isDir bool) os.FileInfo {
return &fileInfo{
name: name,
size: size,
mode: mode,
modTime: modificationTime,
isDir: isDir,
}
}
func NewFileMode(fileAttributes string) (os.FileMode, error) {
var result os.FileMode
if len(fileAttributes) != 10 {
return result, fmt.Errorf("Invalid attribute length %v %v", fileAttributes, len(fileAttributes))
}
const fileType = "dalTLDpSugct"
var fileModePosition = strings.Index(fileType, string(fileAttributes[0]))
if fileModePosition != -1 {
result = 1 << uint(32-1-fileModePosition)
}
const filePermission = "rwxrwxrwx"
for i, c := range filePermission {
if c == rune(fileAttributes[i+1]) {
result = result | 1< 0 {
query.Prefix = parsedUrl.Path[1:]
}
responseIterator := client.Bucket(parsedUrl.Host).Objects(ctx, query)
var result = make([]tstorage.Object, 0)
for obj, err := responseIterator.Next(); err == nil; obj, err = responseIterator.Next() {
objectURL := "gs://" + parsedUrl.Host + "/" + obj.Prefix + obj.Name
var fileMode, _ = tstorage.NewFileMode("-rw-rw-rw-")
if obj.Prefix != "" {
fileMode, _ = tstorage.NewFileMode("drw-rw-rw-")
}
var _, name = toolbox.URLSplit(objectURL)
var fileInfo = tstorage.NewFileInfo(name, obj.Size, fileMode, obj.Updated, fileMode.IsDir())
var object = newStorageObject(objectURL, obj, fileInfo)
result = append(result, object)
}
return result, err
}
func (s *service) Exists(URL string) (bool, error) {
objects, err := s.List(URL)
if err != nil {
return false, err
}
return len(objects) > 0, nil
}
func (s *service) StorageObject(URL string) (tstorage.Object, error) {
objects, err := s.List(URL)
if err != nil {
return nil, err
}
if len(objects) == 0 {
return nil, fmt.Errorf("Not found %v", URL)
}
return objects[0], nil
}
//Download returns reader for downloaded storage object
func (s *service) Download(object tstorage.Object) (io.ReadCloser, error) {
client, ctx, err := s.NewClient()
if err != nil {
return nil, err
}
defer client.Close()
objectInfo := &storage.ObjectAttrs{}
err = object.Unwrap(&objectInfo)
if err != nil {
return nil, err
}
return client.Bucket(objectInfo.Bucket).
Object(objectInfo.Name).
NewReader(ctx)
}
func (s *service) Upload(URL string, reader io.Reader) error {
return s.UploadWithMode(URL, tstorage.DefaultFileMode, reader)
}
func (s *service) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
parserURL, err := url.Parse(URL)
if err != nil {
return fmt.Errorf("failed to parse URL for uploading: %v, %v", URL, err)
}
client, ctx, err := s.NewClient()
if err != nil {
return err
}
defer client.Close()
name := parserURL.Path
if len(parserURL.Path) > 0 {
name = parserURL.Path[1:]
}
err = s.uploadContent(ctx, client, parserURL, name, reader)
if toolbox.IsNotFoundError(err) {
err := client.Bucket(parserURL.Host).Create(ctx, s.projectID, &storage.BucketAttrs{})
if err != nil {
return fmt.Errorf("failed to create bucket: %v, %v", parserURL.Host, err)
}
//_, _ = client.Bucket(parserURL.Host).DefaultObjectACL().List(ctx)
return s.uploadContent(ctx, client, parserURL, name, reader)
}
if err != nil {
return fmt.Errorf("unable upload: %v", err)
}
return nil
}
func (s *service) uploadContent(ctx context.Context, client *storage.Client, parserURL *url.URL, name string, reader io.Reader) error {
writer := client.Bucket(parserURL.Host).
Object(name).
NewWriter(ctx)
expiry := parserURL.Query().Get("expiry")
if expiry != "" {
writer.Metadata = map[string]string{
"Cache-Control": "private, max-age=" + expiry,
}
}
reader, err := updateUploadChecksum(parserURL, reader, writer)
if _, err = io.Copy(writer, reader); err != nil {
return fmt.Errorf("failed to copy to writer during upload:%v", err)
}
if err = writer.Close(); err != nil {
return toolbox.ReclassifyNotFoundIfMatched(err, parserURL.String())
}
return nil
}
func updateUploadChecksum(parserURL *url.URL, reader io.Reader, writer *storage.Writer) (io.Reader, error) {
checksumDisabled := parserURL.Query().Get("disableChecksum") != ""
updateMD5 := parserURL.Query().Get("disableMD5") == ""
updateCRC := parserURL.Query().Get("disableCRC32") == ""
if !(updateCRC || updateMD5) || checksumDisabled {
return reader, nil
}
var err error
bufferReader, ok := reader.(*bytes.Buffer)
if !ok {
content, err := ioutil.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to read all during upload:%v", err)
}
bufferReader = bytes.NewBuffer(content)
}
if parserURL.Query().Get("disableMD5") == "" {
hashReader := bytes.NewBuffer(bufferReader.Bytes())
h := md5.New()
_, _ = io.Copy(h, hashReader)
writer.MD5 = h.Sum(nil)
hashReader.Reset()
} else if parserURL.Query().Get("disableCRC32") == "" {
crc32HashReader := bytes.NewBuffer(bufferReader.Bytes())
crc32Hash := crc32.New(crc32.MakeTable(crc32.Castagnoli))
_, _ = io.Copy(crc32Hash, crc32HashReader)
writer.CRC32C = crc32Hash.Sum32()
crc32HashReader.Reset()
}
return bufferReader, err
}
func (s *service) Register(schema string, service tstorage.Service) error {
return errors.New("unsupported")
}
func (s *service) Close() error {
return nil
}
func (s *service) listAll(URL string, result *[]tstorage.Object) error {
if !strings.HasSuffix(URL, "/") {
URL += "/"
}
objects, err := s.List(URL)
if err != nil {
return err
}
for _, object := range objects {
if !object.IsFolder() {
*result = append(*result, object)
continue
}
if err = s.listAll(object.URL(), result); err != nil {
return err
}
}
return nil
}
//Delete removes passed in storage object
func (s *service) Delete(object tstorage.Object) error {
client, ctx, err := s.NewClient()
if err != nil {
return err
}
defer client.Close()
objectInfo := &storage.ObjectAttrs{}
err = object.Unwrap(&objectInfo)
if err != nil {
return err
}
if object.IsFolder() {
var objects = []tstorage.Object{}
err := s.listAll(object.URL(), &objects)
if err != nil {
return err
}
for _, object := range objects {
if err := s.Delete(object); err != nil {
return err
}
}
return nil
}
return client.Bucket(objectInfo.Bucket).
Object(objectInfo.Name).Delete(ctx)
}
//DownloadWithURL downloads content for passed in object URL
func (s *service) DownloadWithURL(URL string) (io.ReadCloser, error) {
object, err := s.StorageObject(URL)
if err != nil {
return nil, err
}
return s.Download(object)
}
//NewService create a new gc storage service
func NewService(projectId string, options ...option.ClientOption) *service {
return &service{
projectID: projectId,
options: options,
}
}
toolbox-0.33.2/storage/gs/service_test.go 0000664 0000000 0000000 00000000770 13741102511 0020367 0 ustar 00root root 0000000 0000000 package gs
import (
"testing"
)
func TestService_List(t *testing.T) {
/* credential := option.WithServiceAccountFile("")
service := gs.NewService(credential)
assert.NotNil(t, service)
objects, err := service.List("")
assert.Nil(t, err)
for _, o := range objects {
fmt.Printf("%v\n", o.URL())
}
object, err := service.StorageObject("")
assert.Nil(t, err)
assert.NotNil(t, object)
err = service.Delete(object)
assert.Nil(t, err)*/
}
toolbox-0.33.2/storage/http_service.go 0000664 0000000 0000000 00000017614 13741102511 0017763 0 ustar 00root root 0000000 0000000 package storage
import (
"fmt"
"github.com/pkg/errors"
"github.com/viant/toolbox"
"github.com/viant/toolbox/cred"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
)
//httpStorageService represents basic http storage service (only limited listing and full download are supported)
type httpStorageService struct {
Credential *cred.Config
}
//HTTPClientProvider represents http client provider
var HTTPClientProvider = func() (*http.Client, error) {
return toolbox.NewHttpClient(&toolbox.HttpOptions{Key: "MaxIdleConns", Value: 0})
}
func (s *httpStorageService) addCredentialToURLIfNeeded(URL string) string {
if s.Credential == nil || s.Credential.Password == "" || s.Credential.Username == "" {
return URL
}
prasedURL, err := url.Parse(URL)
if err != nil {
return URL
}
if prasedURL.User != nil {
return URL
}
return strings.Replace(URL, "://", fmt.Sprintf("://%v:%v@", s.Credential.Username, s.Credential.Password), 1)
}
type hRef struct {
URL string
Value string
}
func extractLinks(body string) []*hRef {
var result = make([]*hRef, 0)
var linkContents = strings.Split(string(body), "href=\"")
for i := 1; i < len(linkContents); i++ {
var linkContent = linkContents[i]
linkEndPosition := strings.Index(linkContent, "\"")
if linkEndPosition == -1 {
continue
}
linkHref := string(linkContent[:linkEndPosition])
var content = ""
contentStartPosition := strings.Index(linkContent, ">")
if contentStartPosition != 1 {
content = string(linkContent[contentStartPosition+1:])
contentEndPosition := strings.Index(content, "<")
if contentEndPosition != -1 {
content = string(content[:contentEndPosition])
}
}
link := &hRef{
URL: linkHref,
Value: strings.Trim(content, " \t\r\n"),
}
result = append(result, link)
}
return result
}
//List returns a list of object for supplied url
func (s *httpStorageService) List(URL string) ([]Object, error) {
listURL := s.addCredentialToURLIfNeeded(URL)
client, err := HTTPClientProvider()
if err != nil {
return nil, err
}
response, err := client.Get(listURL)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}
now := time.Now()
contentType := response.Header.Get("Content-Type")
var result = make([]Object, 0)
if response.Status != "200 OK" {
return nil, fmt.Errorf("Invalid response code: %v", response.Status)
}
isGitUrl := strings.Contains(URL, "github.")
isPublicGit := strings.Contains(URL, "github.com")
if strings.Contains(contentType, "text/html") {
links := extractLinks(string(body))
var indexedLinks = map[string]bool{}
if isGitUrl {
for _, link := range links {
if !((strings.Contains(link.URL, "/blob/") || strings.Contains(link.URL, "/tree/")) && strings.HasSuffix(link.URL, link.Value)) {
continue
}
linkType := StorageObjectContentType
_, name := toolbox.URLSplit(link.URL)
if path.Ext(name) == "" {
linkType = StorageObjectFolderType
}
baseURL := toolbox.URLBase(URL)
objectURL := link.URL
if !strings.Contains(objectURL, baseURL) {
objectURL = toolbox.URLPathJoin(baseURL, link.URL)
}
if linkType == StorageObjectContentType && strings.Contains(objectURL, "/master/") {
objectURL = strings.Replace(objectURL, "/blob/", "/", 1)
if isPublicGit {
objectURL = strings.Replace(objectURL, "github.com", "raw.githubusercontent.com", 1)
} else {
objectURL = strings.Replace(objectURL, ".com/", ".com/raw/", 1)
}
}
if linkType == StorageObjectContentType && !strings.Contains(objectURL, "raw") {
continue
}
if _, ok := indexedLinks[objectURL]; ok {
continue
}
storageObject := newHttpFileObject(objectURL, linkType, nil, now, 1)
indexedLinks[objectURL] = true
result = append(result, storageObject)
}
} else {
for _, link := range links {
if link.URL == "" || strings.Contains(link.URL, ":") || strings.HasPrefix(link.URL, "#") || strings.HasPrefix(link.URL, "?") || strings.HasPrefix(link.URL, ".") || strings.HasPrefix(link.URL, "/") {
continue
}
linkType := StorageObjectContentType
if strings.HasSuffix(link.URL, "/") {
linkType = StorageObjectFolderType
}
objectURL := toolbox.URLPathJoin(URL, link.URL)
storageObject := newHttpFileObject(objectURL, linkType, nil, now, 1)
result = append(result, storageObject)
}
}
}
if strings.Contains(string(body), ">..<") {
return result, err
}
storageObject := newHttpFileObject(URL, StorageObjectContentType, nil, now, response.ContentLength)
result = append(result, storageObject)
return result, err
}
//Exists returns true if resource exists
func (s *httpStorageService) Exists(URL string) (bool, error) {
client, err := HTTPClientProvider()
if err != nil {
return false, err
}
response, err := client.Get(URL)
if err != nil {
return false, err
}
return response.StatusCode == 200, nil
}
//Object returns a Object for supplied url
func (s *httpStorageService) StorageObject(URL string) (Object, error) {
objects, err := s.List(URL)
if err != nil {
return nil, err
}
if len(objects) == 0 {
return nil, fmt.Errorf("resource not found: %v", URL)
}
return objects[0], nil
}
//Download returns reader for downloaded storage object
func (s *httpStorageService) Download(object Object) (io.ReadCloser, error) {
client, err := HTTPClientProvider()
if err != nil {
return nil, err
}
response, err := client.Get(s.addCredentialToURLIfNeeded(object.URL()))
return response.Body, err
}
//Upload uploads provided reader content for supplied url.
func (s *httpStorageService) Upload(URL string, reader io.Reader) error {
return errors.New("unsupported")
}
//Upload uploads provided reader content for supplied url.
func (s *httpStorageService) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
return errors.New("unsupported")
}
func (s *httpStorageService) Register(schema string, service Service) error {
return errors.New("unsupported")
}
//Delete removes passed in storage object
func (s *httpStorageService) Delete(object Object) error {
fileName := toolbox.Filename(object.URL())
return os.Remove(fileName)
}
func (s *httpStorageService) Close() error {
return nil
}
//DownloadWithURL downloads content for passed in object URL
func (s *httpStorageService) DownloadWithURL(URL string) (io.ReadCloser, error) {
object, err := s.StorageObject(URL)
if err != nil {
return nil, err
}
return s.Download(object)
}
func NewHttpStorageService(credential *cred.Config) Service {
return &httpStorageService{
Credential: credential,
}
}
type httpStorageObject struct {
*AbstractObject
}
func (o *httpStorageObject) Unwrap(target interface{}) error {
return fmt.Errorf("unsuported target %T", target)
}
func newHttpFileObject(url string, objectType int, source interface{}, lastModified time.Time, size int64) Object {
var isDir = objectType == StorageObjectFolderType
var _, name = toolbox.URLSplit(url)
var fileMode, _ = NewFileMode("-r--r--r--")
if isDir {
fileMode, _ = NewFileMode("dr--r--r--")
}
fileInfo := NewFileInfo(name, size, fileMode, lastModified, isDir)
abstract := NewAbstractStorageObject(url, source, fileInfo)
result := &httpStorageObject{
AbstractObject: abstract,
}
result.AbstractObject.Object = result
return result
}
const HttpProviderScheme = "http"
const HttpsProviderScheme = "https"
func init() {
Registry().Registry[HttpsProviderScheme] = httpServiceProvider
Registry().Registry[HttpProviderScheme] = httpServiceProvider
}
func httpServiceProvider(credentialFile string) (Service, error) {
if credentialFile == "" {
return NewHttpStorageService(nil), nil
}
if !strings.HasPrefix(credentialFile, "/") {
dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
credentialFile = path.Join(dir, credentialFile)
}
config, err := cred.NewConfig(credentialFile)
if err != nil {
return nil, err
}
return NewHttpStorageService(config), nil
}
toolbox-0.33.2/storage/http_service_test.go 0000664 0000000 0000000 00000001134 13741102511 0021010 0 ustar 00root root 0000000 0000000 package storage_test
import "testing"
func TestNewHttpStorageService(t *testing.T) {
// Deprecated use gith ub/viant/afs/http
//{
// service, err := storage.NewServiceForURL("https://github.com/viant/", credentialFile)
// assert.Nil(t, err)
// assert.NotNil(t, service)
//
// objects, err := service.List("https://github.com/viant/")
// assert.True(t, len(objects) > 0)
// reader, err := service.Download(objects[0])
// if assert.Nil(t, err) {
// defer reader.Close()
// }
// content, err := ioutil.ReadAll(reader)
// assert.Nil(t, err)
// assert.True(t, len(content) > 0)
//
//}
}
toolbox-0.33.2/storage/mapper.go 0000664 0000000 0000000 00000006615 13741102511 0016547 0 ustar 00root root 0000000 0000000 package storage
import (
"fmt"
"github.com/viant/toolbox"
"io"
"io/ioutil"
"os"
"path"
"strings"
)
type StorageMapping struct {
SourceURL string
SourceCredential string
DestinationURI string
TargetFile string
TargetPackage string
UseTextFormat bool
}
var binaryFormats = map[string]bool{
".png": true,
".jpeg": true,
".ico": true,
".jpg": true,
}
//GenerateStorageCode create a *.go files with statically scanned content from source URL.
func GenerateStorageCode(mappings ...*StorageMapping) error {
destinationService := NewMemoryService()
for _, mapping := range mappings {
sourceService, err := NewServiceForURL(mapping.SourceURL, mapping.SourceCredential)
if err != nil {
return err
}
handler, writer, err := NewStorageMapperHandler(mapping.TargetFile, mapping.TargetPackage, mapping.UseTextFormat, binaryFormats)
if err != nil {
return err
}
defer writer.Close()
destinationURL := "mem://" + mapping.DestinationURI
err = copyStorageContent(sourceService, mapping.SourceURL, destinationService, destinationURL, nil, "", handler)
if err != nil {
return err
}
}
return nil
}
//NewStorageMapperHandler creates a template handler for generating go file that write static content into memory service.
func NewStorageMapperHandler(filename, pkg string, useTextFormat bool, binaryFormat map[string]bool) (CopyHandler, io.WriteCloser, error) {
_ = toolbox.RemoveFileIfExist(filename)
writer, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, nil, err
}
template := &templateWriter{writer}
_ = template.Init(pkg)
return func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error {
_, name := toolbox.URLSplit(destinationURL)
if strings.HasPrefix(name, ".") {
//skip hidden files
return nil
}
content, err := ioutil.ReadAll(source)
if err != nil {
return err
}
isText := useTextFormat
ext := path.Ext(destinationURL)
if binaryFormat[ext] {
isText = false
}
template.WriteStorageContent(destinationURL, content, isText)
return nil
}, template, nil
}
type templateWriter struct {
io.WriteCloser
}
func (t *templateWriter) Init(pkg string) error {
var begin = `package %v
import (
"bytes"
"github.com/viant/toolbox/storage"
"log"
)
func init() {
var memStorage = storage.NewMemoryService();
`
_, err := t.Write([]byte(fmt.Sprintf(begin, pkg)))
return err
}
func (t *templateWriter) WriteStorageContent(URL string, content []byte, useText bool) error {
if useText {
text := string(content)
var count = strings.Count(text, "`")
if count > 0 {
text = strings.Replace(text, "`", "`+\"`\"+`", count)
content = []byte(text)
}
}
var contentReader = fmt.Sprintf("bytes.NewReader([]byte(`%s`))", content)
if !toolbox.IsASCIIText(contentReader) && !useText {
var byteArray = make([]string, 0)
for _, b := range content {
byteArray = append(byteArray, fmt.Sprintf("%d", b))
}
contentReader = fmt.Sprintf("bytes.NewReader([]byte{%v})", strings.Join(byteArray, ","))
}
var payload = ` {
err := memStorage.Upload("%v", %v)
if err != nil {
log.Printf("failed to upload: %v %v", err)
}
}
`
payload = fmt.Sprintf(payload, URL, contentReader, URL, "%v")
_, err := t.Write([]byte(payload))
return err
}
func (t *templateWriter) Close() error {
var end = "}\n"
_, err := t.Write([]byte(end))
t.WriteCloser.Close()
return err
}
toolbox-0.33.2/storage/mapper_test.go 0000664 0000000 0000000 00000001761 13741102511 0017603 0 ustar 00root root 0000000 0000000 package storage_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
"io/ioutil"
"os"
"path"
"testing"
)
func TestTemplateWriter_GenerateStorageCode(t *testing.T) {
parent := toolbox.CallerDirectory(3)
var source = toolbox.FileSchema + path.Join(parent, "test", "source")
var destination = path.Join("test", "source")
var target = path.Join(parent, "test", "gen", "source.go")
parent, _ = path.Split(target)
toolbox.CreateDirIfNotExist(parent)
err := storage.GenerateStorageCode(&storage.StorageMapping{
SourceURL: source,
DestinationURI: destination,
TargetFile: target,
TargetPackage: "gen",
})
assert.Nil(t, err)
reader, err := os.Open(target)
assert.Nil(t, err)
defer reader.Close()
content, err := ioutil.ReadAll(reader)
if assert.Nil(t, err) {
textContent := string(content)
assert.Contains(t, textContent, `err := memStorage.Upload("mem://test/source/file2.txt", bytes.NewReader([]byte`)
}
}
toolbox-0.33.2/storage/mem_service.go 0000664 0000000 0000000 00000015350 13741102511 0017555 0 ustar 00root root 0000000 0000000 package storage
import (
"bytes"
"errors"
"io"
"io/ioutil"
"net/url"
"os"
"strings"
"sync"
"time"
)
const MemoryProviderScheme = "mem"
var noSuchFileOrDirectoryError = errors.New("No such file or directory")
var folderMode, _ = NewFileMode("drwxrwxrwx")
//MemoryRoot represents memory root storage
var MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))
//ResetMemory reset memory root storage
func ResetMemory() {
MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))
}
//Service represents memory storage service intended for testing
type memoryStorageService struct {
root *MemoryFolder
}
type MemoryFile struct {
name string
fileInfo os.FileInfo
content []byte
}
func (f *MemoryFile) Object() Object {
return NewAbstractStorageObject(f.name, f, f.fileInfo)
}
type MemoryFolder struct {
name string
fileInfo os.FileInfo
mutext *sync.RWMutex
files map[string]*MemoryFile
folders map[string]*MemoryFolder
}
func (f *MemoryFolder) Object() Object {
return NewAbstractStorageObject(f.name, f, f.fileInfo)
}
func (f *MemoryFolder) Objects() []Object {
var result = make([]Object, 0)
result = append(result, f.Object())
for _, folder := range f.folders {
result = append(result, folder.Object())
}
for _, file := range f.files {
result = append(result, file.Object())
}
return result
}
func newMemoryFolder(name string, info os.FileInfo) *MemoryFolder {
return &MemoryFolder{
name: name,
fileInfo: info,
mutext: &sync.RWMutex{},
files: make(map[string]*MemoryFile),
folders: make(map[string]*MemoryFolder),
}
}
func (s *memoryStorageService) getFolder(pathFragments []string) (*MemoryFolder, error) {
node := s.root
var ok bool
for i := 1; i+1 < len(pathFragments); i++ {
pathFragment := pathFragments[i]
node, ok = node.folders[pathFragment]
if !ok {
return nil, noSuchFileOrDirectoryError
}
}
return node, nil
}
func (s *memoryStorageService) getPath(URL string) (string, error) {
parsedURL, err := url.Parse(URL)
if err != nil {
return "", err
}
parsedPath := parsedURL.Path
strings.Replace(parsedPath, "//", "/", len(parsedPath))
if len(parsedPath) > 1 && strings.HasSuffix(parsedPath, "/") {
parsedPath = string(parsedPath[:len(parsedPath)-1])
}
return parsedPath, nil
}
//List returns a list of object for supplied url
func (s *memoryStorageService) List(URL string) ([]Object, error) {
path, err := s.getPath(URL)
if err != nil {
return nil, err
}
if path == "/" {
return s.root.Objects(), nil
}
var pathFragments = strings.Split(path, "/")
node, err := s.getFolder(pathFragments)
if err != nil {
return nil, err
}
var pathLeaf = pathFragments[len(pathFragments)-1]
if memoryFile, ok := node.files[pathLeaf]; ok {
return []Object{memoryFile.Object()}, nil
}
if folder, ok := node.folders[pathLeaf]; ok {
return folder.Objects(), nil
}
return []Object{}, nil
}
//Exists returns true if resource exists
func (s *memoryStorageService) Exists(URL string) (bool, error) {
objects, err := s.List(URL)
if err != nil {
return false, err
}
return len(objects) > 0, nil
}
func (s *memoryStorageService) Close() error {
return nil
}
//Object returns a Object for supplied url
func (s *memoryStorageService) StorageObject(URL string) (Object, error) {
objects, err := s.List(URL)
if err != nil {
return nil, err
}
if len(objects) == 0 {
return nil, noSuchFileOrDirectoryError
}
return objects[0], nil
}
//Download returns reader for downloaded storage object
func (s *memoryStorageService) Download(object Object) (io.ReadCloser, error) {
var urlPath, err = s.getPath(object.URL())
if err != nil {
return nil, err
}
var pathFragments = strings.Split(urlPath, "/")
node, err := s.getFolder(pathFragments)
if err != nil {
return nil, err
}
var pathLeaf = pathFragments[len(pathFragments)-1]
if memoryFile, ok := node.files[pathLeaf]; ok {
return ioutil.NopCloser(bytes.NewReader(memoryFile.content)), nil
}
return nil, noSuchFileOrDirectoryError
}
//Upload uploads provided reader content for supplied url.
func (s *memoryStorageService) Upload(URL string, reader io.Reader) error {
return s.UploadWithMode(URL, DefaultFileMode, reader)
}
//Upload uploads provided reader content for supplied url.
func (s *memoryStorageService) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
urlPath, err := s.getPath(URL)
if err != nil {
return err
}
content, err := ioutil.ReadAll(reader)
if err != nil {
return err
}
var node = s.root
var pathFragments = strings.Split(urlPath, "/")
for i := 1; i+1 < len(pathFragments); i++ {
pathFragment := pathFragments[i]
if subFolder, ok := node.folders[pathFragment]; ok {
node = subFolder
} else {
var folderURL = MemoryProviderScheme + "://" + strings.Join(pathFragments[:i+1], "/")
var folderInfo = NewFileInfo(pathFragment, 102, folderMode, time.Now(), true)
newFolder := newMemoryFolder(folderURL, folderInfo)
node.mutext.Lock()
node.folders[folderInfo.Name()] = newFolder
node.mutext.Unlock()
node = newFolder
}
}
var pathLeaf = pathFragments[len(pathFragments)-1]
fileInfo := NewFileInfo(pathLeaf, int64(len(content)), fileMode, time.Now(), false)
var memoryFile = &MemoryFile{name: URL, content: content, fileInfo: fileInfo}
node.files[fileInfo.Name()] = memoryFile
return nil
}
func (s *memoryStorageService) Register(schema string, service Service) error {
return errors.New("unsupported")
}
//Delete removes passed in storage object
func (s *memoryStorageService) Delete(object Object) error {
var urlPath, err = s.getPath(object.URL())
if err != nil {
return err
}
var pathFragments = strings.Split(urlPath, "/")
node, err := s.getFolder(pathFragments)
if err != nil {
return err
}
var pathLeaf = pathFragments[len(pathFragments)-1]
if _, ok := node.files[pathLeaf]; ok {
delete(node.files, pathLeaf)
return nil
}
if _, ok := node.files[pathLeaf]; ok {
delete(node.folders, pathLeaf)
return nil
}
return noSuchFileOrDirectoryError
}
//DownloadWithURL downloads content for passed in object URL
func (s *memoryStorageService) DownloadWithURL(URL string) (io.ReadCloser, error) {
object, err := s.StorageObject(URL)
if err != nil {
return nil, err
}
return s.Download(object)
}
// creates a new private memory service
func NewPrivateMemoryService() Service {
return &memoryStorageService{
root: newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true)),
}
}
//creates a new memory service
func NewMemoryService() Service {
return &memoryStorageService{
root: MemoryRoot,
}
}
func init() {
Registry().Registry[MemoryProviderScheme] = memServiceProvider
}
func memServiceProvider(credentialFile string) (Service, error) {
return NewMemoryService(), nil
}
toolbox-0.33.2/storage/mem_service_test.go 0000664 0000000 0000000 00000006024 13741102511 0020612 0 ustar 00root root 0000000 0000000 package storage_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
"io/ioutil"
"strings"
"testing"
)
func Test_NewMemoryService(t *testing.T) {
storage.ResetMemory()
service := storage.NewMemoryService()
var files = map[string]string{
"mem:///test/file1.txt": "abc",
"mem:///test/file2.txt": "xyz",
"mem:///test/sub/file1.txt": "---",
"mem:///test/sub/file2.txt": "xxx",
}
for k, v := range files {
err := service.Upload(k, strings.NewReader(v))
assert.Nil(t, err)
}
for k, v := range files {
object, err := service.StorageObject(k)
if assert.Nil(t, err) {
reader, err := service.Download(object)
if assert.Nil(t, err) {
defer reader.Close()
content, err := ioutil.ReadAll(reader)
assert.Nil(t, err)
assert.Equal(t, v, string(content))
}
}
}
{
objects, err := service.List("mem:///")
assert.Nil(t, err)
assert.Equal(t, 2, len(objects))
assert.True(t, objects[0].IsFolder())
assert.EqualValues(t, "mem:///", objects[0].URL())
assert.True(t, objects[1].IsFolder())
assert.EqualValues(t, "mem:///test", objects[1].URL())
}
{
objects, err := service.List("mem:///test")
assert.Nil(t, err)
assert.Equal(t, 4, len(objects))
assert.True(t, objects[0].IsFolder())
assert.EqualValues(t, "mem:///test", objects[0].URL())
assert.True(t, objects[1].IsFolder())
assert.EqualValues(t, "mem:///test/sub", objects[1].URL())
}
{
objects, err := service.List("mem:///test/sub/")
assert.Nil(t, err)
assert.Equal(t, 3, len(objects))
assert.True(t, objects[0].IsFolder())
for k := range files {
object, err := service.StorageObject(k)
if assert.Nil(t, err) {
err = service.Delete(object)
assert.Nil(t, err)
}
}
}
}
func TestMemCopy(t *testing.T) {
service := storage.NewMemoryService()
defer service.Close()
var files = map[string]string{
"mem:///test/file1.txt": "abc",
"mem:///test/file2.txt": "xyz",
"mem:///test/sub/file1.txt": "---",
"mem:///test/sub/file2.txt": "xxx",
}
for k, v := range files {
err := service.Upload(k, strings.NewReader(v))
assert.Nil(t, err)
}
for k, v := range files {
object, err := service.StorageObject(k)
if assert.Nil(t, err) {
reader, err := service.Download(object)
if assert.Nil(t, err) {
defer reader.Close()
content, err := ioutil.ReadAll(reader)
assert.Nil(t, err)
assert.Equal(t, v, string(content))
}
}
}
baseUrl := "mem://"
sourceURL := toolbox.URLPathJoin(baseUrl, "/test/")
targetURL := toolbox.URLPathJoin(baseUrl, "/target")
err := storage.Copy(service, sourceURL, service, targetURL, nil, nil)
assert.Nil(t, err)
for k, v := range files {
k = strings.Replace(k, "mem:///test/", "mem:///target/", 1)
object, err := service.StorageObject(k)
if assert.Nil(t, err) {
reader, err := service.Download(object)
if assert.Nil(t, err, k) {
defer reader.Close()
content, err := ioutil.ReadAll(reader)
assert.Nil(t, err)
assert.Equal(t, v, string(content))
}
}
}
}
toolbox-0.33.2/storage/object.go 0000664 0000000 0000000 00000003557 13741102511 0016533 0 ustar 00root root 0000000 0000000 package storage
import (
"os"
)
const (
undefined int = iota
StorageObjectFolderType //folder type
StorageObjectContentType //file type
)
//Object represents a storage object
type Object interface {
//URL return storage url
URL() string
//Type returns storage type either folder or file
Type() int
//IsFolder returns true if object is a folder
IsFolder() bool
//IsContent returns true if object is a file
IsContent() bool
//Wrap wraps source storage object
Wrap(source interface{})
//Unwrap unwraps source storage object into provided target.
Unwrap(target interface{}) error
FileInfo() os.FileInfo
}
//AbstractObject represents abstract storage object
type AbstractObject struct {
Object
url string
objectType int
Source interface{}
fileInfo os.FileInfo
}
//URL return storage url
func (o *AbstractObject) URL() string {
return o.url
}
//Type returns storage type StorageObjectFolderType or StorageObjectContentType
func (o *AbstractObject) Type() int {
return o.objectType
}
//IsFolder returns true if object is a folder
func (o *AbstractObject) IsFolder() bool {
return o.objectType == StorageObjectFolderType
}
//IsContent returns true if object is a file
func (o *AbstractObject) IsContent() bool {
return o.objectType == StorageObjectContentType
}
//Wrap wraps source storage object
func (o *AbstractObject) Wrap(source interface{}) {
o.Source = source
}
func (o *AbstractObject) FileInfo() os.FileInfo {
return o.fileInfo
}
//NewAbstractStorageObject creates a new abstract storage object
func NewAbstractStorageObject(url string, source interface{}, fileInfo os.FileInfo) *AbstractObject {
var result = &AbstractObject{
url: url,
Source: source,
fileInfo: fileInfo,
objectType: StorageObjectContentType,
}
if fileInfo.IsDir() {
result.objectType = StorageObjectFolderType
}
return result
}
toolbox-0.33.2/storage/provider.go 0000664 0000000 0000000 00000000165 13741102511 0017107 0 ustar 00root root 0000000 0000000 package storage
//Provider represetns a service provider
type Provider func(credentialFile string) (Service, error)
toolbox-0.33.2/storage/registry.go 0000664 0000000 0000000 00000000702 13741102511 0017122 0 ustar 00root root 0000000 0000000 package storage
type registry struct {
Registry map[string]Provider
}
func (p *registry) Get(namespace string) func(credentialFile string) (Service, error) {
return p.Registry[namespace]
}
var registrySingleton *registry
//Registry returns new provider
func Registry() *registry {
if registrySingleton != nil {
return registrySingleton
}
registrySingleton = ®istry{
Registry: make(map[string]Provider),
}
return registrySingleton
}
toolbox-0.33.2/storage/s3/ 0000775 0000000 0000000 00000000000 13741102511 0015251 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/s3/object.go 0000664 0000000 0000000 00000001772 13741102511 0017055 0 ustar 00root root 0000000 0000000 package s3
import (
"fmt"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/viant/toolbox/storage"
"os"
)
type object struct {
*storage.AbstractObject
}
func (o *object) Unwrap(target interface{}) error {
if commonPrefix, casted := target.(**s3.CommonPrefix); casted {
source, ok := o.Source.(*s3.CommonPrefix)
if !ok {
return fmt.Errorf("failed to case %T into %T", o.Source, target)
}
*commonPrefix = source
}
if commonPrefix, casted := target.(**s3.Object); casted {
source, ok := o.Source.(*s3.Object)
if !ok {
return fmt.Errorf("failed to case %T into %T", o.Source, target)
}
*commonPrefix = source
}
return fmt.Errorf("unsuported target %T", target)
}
//newStorageObject creates a new aws storage object
func newStorageObject(url string, source interface{}, fileInfo os.FileInfo) storage.Object {
abstract := storage.NewAbstractStorageObject(url, source, fileInfo)
result := &object{
AbstractObject: abstract,
}
result.AbstractObject.Object = result
return result
}
toolbox-0.33.2/storage/s3/provider.go 0000664 0000000 0000000 00000002023 13741102511 0017427 0 ustar 00root root 0000000 0000000 package s3
import (
"github.com/viant/toolbox/cred"
"os"
"path"
"path/filepath"
"strings"
"github.com/viant/toolbox/storage"
"github.com/viant/toolbox/url"
)
const ProviderScheme = "s3"
func init() {
SetDefaultProvider()
}
func serviceProvider(credentialFile string) (storage.Service, error) {
s3config := &cred.Config{}
if credentialFile != "" {
if !strings.HasPrefix(credentialFile, "/") {
dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
credentialFile = path.Join(dir, credentialFile)
}
resource := url.NewResource(credentialFile)
err := resource.Decode(s3config)
if err != nil {
return nil, err
}
}
return NewService(s3config), nil
}
//SetProvider set s3 provider with dynamic credentials
func SetDefaultProvider() {
storage.Registry().Registry[ProviderScheme] = serviceProvider
}
//SetProvider set s3 provider with supplied config
func SetProvider(config *cred.Config) {
storage.Registry().Registry[ProviderScheme] = func(string) (storage.Service, error) {
return NewService(config), nil
}
}
toolbox-0.33.2/storage/s3/provider_test.go 0000664 0000000 0000000 00000001137 13741102511 0020473 0 ustar 00root root 0000000 0000000 package s3
import (
"path"
"testing"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
)
const s3Url = "s3://path"
const s3Secret = "mock_s3.json"
//Troubleshooting prod issue with credentials not loading
func TestLoadingCredentialFile(t *testing.T) {
// Creating absolute path
fileName, _, _ := toolbox.CallerInfo(2)
currentPath, _ := path.Split(fileName)
credentialPath := path.Dir(path.Dir(currentPath)) + "/test/" + s3Secret
service, err := storage.NewServiceForURL(s3Url, credentialPath)
assert.Nil(t, err)
assert.NotNil(t, service)
}
toolbox-0.33.2/storage/s3/service.go 0000664 0000000 0000000 00000015515 13741102511 0017247 0 ustar 00root root 0000000 0000000 package s3
import (
"bytes"
"fmt"
"github.com/viant/toolbox/cred"
"io"
"net/url"
"strings"
"io/ioutil"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
"os"
)
var defaultTime = time.Time{}
type service struct {
config *cred.Config
}
func listFolders(client *s3.S3, url *url.URL, result *[]storage.Object) error {
folderRequest := &s3.ListObjectsInput{
Bucket: aws.String(url.Host),
Prefix: aws.String(url.Path[1:]),
Delimiter: aws.String("/"),
}
prefixes := make([]*s3.CommonPrefix, 0)
err := client.ListObjectsPages(folderRequest,
func(page *s3.ListObjectsOutput, lastPage bool) bool {
prefixes = append(prefixes, page.CommonPrefixes...)
return len(page.CommonPrefixes) > 0
})
if err != nil {
if strings.Contains(err.Error(), "BucketRegionError") {
return nil
}
return err
}
for _, prefix := range prefixes {
pathURL := "s3://" + url.Host + "/" + *prefix.Prefix
var _, name = toolbox.URLSplit(pathURL)
var fileMode, _ = storage.NewFileMode("drw-rw-rw-")
var fileInfo = storage.NewFileInfo(name, 102, fileMode, defaultTime, fileMode.IsDir())
var object = newStorageObject(pathURL, prefix, fileInfo)
*result = append(*result, object)
}
return nil
}
func listContent(client *s3.S3, parsedURL *url.URL, result *[]storage.Object) error {
var path = parsedURL.Path
folderRequest := &s3.ListObjectsInput{
Bucket: aws.String(parsedURL.Host),
Delimiter: aws.String("/"),
}
if len(path) > 0 {
folderRequest.Prefix = aws.String(parsedURL.Path[1:])
}
contents := make([]*s3.Object, 0)
err := client.ListObjectsPages(folderRequest,
func(page *s3.ListObjectsOutput, lastPage bool) bool {
contents = append(contents, page.Contents...)
return len(page.Contents) > 0
})
if err != nil {
if strings.Contains(err.Error(), "BucketRegionError") {
return nil
}
return err
}
for _, content := range contents {
objectURL := "s3://" + parsedURL.Host + "/" + *content.Key
var _, name = toolbox.URLSplit(objectURL)
var fileMode, _ = storage.NewFileMode("-rw-rw-rw-")
var fileInfo = storage.NewFileInfo(name, *content.Size, fileMode, *content.LastModified, fileMode.IsDir())
var object = newStorageObject(objectURL, content, fileInfo)
*result = append(*result, object)
}
return nil
}
func (s *service) getAwsConfig() (*aws.Config, error) {
if s.config.Secret == "" {
return aws.NewConfig().WithRegion(s.config.Region), nil
}
awsCredentials := credentials.NewStaticCredentials(s.config.Key, s.config.Secret, s.config.Token)
_, err := awsCredentials.Get()
if err != nil {
return nil, fmt.Errorf("bad credentials: %s", err)
}
return aws.NewConfig().WithRegion(s.config.Region).WithCredentials(awsCredentials), nil
}
func (s *service) List(URL string) ([]storage.Object, error) {
var result = make([]storage.Object, 0)
u, err := url.Parse(URL)
if err != nil {
return nil, fmt.Errorf("failed to parse : %v", err)
}
config, err := s.getAwsConfig()
if err != nil {
return nil, fmt.Errorf("failed to get aws config: %v", err)
}
client := s3.New(session.New(), config)
err = listFolders(client, u, &result)
if err == nil {
err = listContent(client, u, &result)
}
if err != nil {
return nil, fmt.Errorf("failed to get list content: %v", err)
}
return result, nil
}
func (s *service) Exists(URL string) (bool, error) {
objects, err := s.List(URL)
if err != nil {
return false, err
}
return len(objects) > 0, nil
}
func (s *service) StorageObject(URL string) (storage.Object, error) {
objects, err := s.List(URL)
if err != nil {
return nil, err
}
if len(objects) == 0 {
return nil, fmt.Errorf("Not found %v", URL)
}
return objects[0], nil
}
func (s *service) Download(object storage.Object) (io.ReadCloser, error) {
u, err := url.Parse(object.URL())
if err != nil {
return nil, fmt.Errorf("failed to parse : %v", err)
}
config, err := s.getAwsConfig()
if err != nil {
return nil, fmt.Errorf("failed to get aws config: %v", err)
}
downloader := s3manager.NewDownloader(session.New(config))
target := &s3.Object{}
_ = object.Unwrap(&target)
writer := toolbox.NewByteWriterAt()
_, err = downloader.Download(writer,
&s3.GetObjectInput{
Bucket: aws.String(u.Host),
Key: aws.String(*target.Key),
})
if err != nil {
return nil, fmt.Errorf("failed to download: %v", err)
}
return ioutil.NopCloser(bytes.NewReader(writer.Buffer)), nil
}
func (s *service) Upload(URL string, reader io.Reader) error {
return s.UploadWithMode(URL, storage.DefaultFileMode, reader)
}
func (s *service) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
err := s.uploadContent(URL, reader)
if toolbox.IsNotFoundError(err) {
config, err := s.getAwsConfig()
if err != nil {
return err
}
if parserURL, err := url.Parse(URL); err == nil {
client := s3.New(session.New(config))
if _, err := client.CreateBucket(&s3.CreateBucketInput{
Bucket: &parserURL.Host,
}); err != nil {
return err
}
}
return s.uploadContent(URL, reader)
}
return err
}
func (s *service) uploadContent(URL string, reader io.Reader) error {
parsedURL, err := url.Parse(URL)
if err != nil {
return err
}
config, err := s.getAwsConfig()
if err != nil {
return err
}
uploader := s3manager.NewUploader(session.New(config))
_, err = uploader.Upload(&s3manager.UploadInput{
Body: reader,
Bucket: aws.String(parsedURL.Host),
Key: aws.String(parsedURL.Path),
})
if err != nil {
return toolbox.ReclassifyNotFoundIfMatched(err, URL)
}
return nil
}
func (s *service) Delete(object storage.Object) error {
parsedURL, err := url.Parse(object.URL())
if err != nil {
return err
}
if object.IsFolder() {
var objects = []storage.Object{}
objects, err = s.List(object.URL())
if err != nil {
return err
}
for _, object := range objects {
if err := s.Delete(object); err != nil {
return err
}
}
return nil
}
target := &s3.Object{}
object.Unwrap(&target)
request := &s3.DeleteObjectInput{
Bucket: aws.String(parsedURL.Host),
Key: target.Key,
}
config, err := s.getAwsConfig()
if err != nil {
return err
}
client := s3.New(session.New(), config)
client.DeleteObject(request)
return nil
}
func (s *service) Register(schema string, service storage.Service) error {
return fmt.Errorf("Unsupported")
}
//DownloadWithURL downloads content for passed in object URL
func (s *service) DownloadWithURL(URL string) (io.ReadCloser, error) {
object, err := s.StorageObject(URL)
if err != nil {
return nil, err
}
return s.Download(object)
}
func (s *service) Close() error {
return nil
}
//NewService creates a new aws storage service
func NewService(config *cred.Config) storage.Service {
return &service{config: config}
}
toolbox-0.33.2/storage/s3/service_test.go 0000664 0000000 0000000 00000001014 13741102511 0020273 0 ustar 00root root 0000000 0000000 package s3_test
import (
"testing"
)
func TestService_List(t *testing.T) {
/* fmt.Print("Has service\n")
config := &aws.Config{
Region: "",
Key: "",
Secret: "",
}
service := aws.NewService(config)
assert.NotNil(t, service)
result, err := service.List("")
assert.Nil(t, err)
for _, o := range result {
fmt.Printf("R: %v\n", o.URL())
}
obj, err := service.StorageObject("")
assert.Nil(t, err)
assert.NotNil(t, obj)
assert.Nil(t, service.Delete(obj))*/
}
toolbox-0.33.2/storage/scp/ 0000775 0000000 0000000 00000000000 13741102511 0015511 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/scp/fileinfo_parser.go 0000664 0000000 0000000 00000013545 13741102511 0021217 0 ustar 00root root 0000000 0000000 package scp
import (
"fmt"
"github.com/lunixbochs/vtclean"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
"net/url"
"strings"
"time"
"unicode"
)
const (
fileInfoPermission = iota
_
fileInfoOwner
fileInfoGroup
fileInfoSize
fileInfoDateMonth
fileInfoDateDay
fileInfoDateHour
fileInfoDateYear
fileInfoName
)
const (
fileIsoInfoPermission = iota
_
fileIsoInfoOwner
fileIsoInfoGroup
fileIsoInfoSize
fileIsoDate
fileIsoTime
fileIsoTimezone
fileIsoInfoName
)
//Parser represents fileinfo parser from stdout
type Parser struct {
IsoTimeStyle bool
}
func (p *Parser) Parse(parsedURL *url.URL, stdout string, isURLDir bool) ([]storage.Object, error) {
var err error
var result = make([]storage.Object, 0)
if strings.Contains(stdout, "No such file or directory") {
return result, nil
}
for _, line := range strings.Split(stdout, "\n") {
if line == "" {
continue
}
var object storage.Object
if p.IsoTimeStyle {
if object, err = p.extractObjectFromIsoBasedTimeCommand(parsedURL, line, isURLDir); err != nil {
object, err = p.extractObjectFromNonIsoBaseTimeCommand(parsedURL, line, isURLDir)
}
} else {
if object, err = p.extractObjectFromNonIsoBaseTimeCommand(parsedURL, line, isURLDir); err != nil {
object, err = p.extractObjectFromIsoBasedTimeCommand(parsedURL, line, isURLDir)
}
}
if err != nil {
return nil, err
}
result = append(result, object)
}
return result, nil
}
func (p *Parser) HasNextTokenInout(nextTokenPosition int, line string) bool {
if nextTokenPosition >= len(line) {
return false
}
nextToken := []rune(string(line[nextTokenPosition:]))[0]
return !unicode.IsSpace(nextToken)
}
func (p *Parser) newObject(parsedURL *url.URL, name, permission, line, size string, modificationTime time.Time, isURLDirectory bool) (storage.Object, error) {
var URLPath = parsedURL.Path
var URL = parsedURL.String()
var pathPosition = strings.Index(URL, parsedURL.Host) + len(parsedURL.Host)
var URLPrefix = URL[:pathPosition]
fileMode, err := storage.NewFileMode(permission)
if err != nil {
return nil, fmt.Errorf("failed to parse line for lineinfo: %v, unable to file attributes: %v", line, err)
}
if isURLDirectory {
name = strings.Replace(name, URLPath, "", 1)
URLPath = toolbox.URLPathJoin(URLPath, name)
} else {
URLPath = name
}
var objectURL = URLPrefix + URLPath
fileInfo := storage.NewFileInfo(name, int64(toolbox.AsInt(size)), fileMode, modificationTime, fileMode.IsDir())
object := newStorageObject(objectURL, fileInfo, fileInfo)
return object, nil
}
//extractObjectFromNonIsoBaseTimeCommand extract file storage object from line,
// it expects a file info without iso i.e -rw-r--r-- 1 awitas 1742120565 414 Jun 8 14:14:08 2017 id_rsa.pub
func (p *Parser) extractObjectFromNonIsoBaseTimeCommand(parsedURL *url.URL, line string, isURLDirectory bool) (storage.Object, error) {
tokenIndex := 0
if strings.TrimSpace(line) == "" {
return nil, nil
}
var owner, name, permission, group, size, year, month, day, hour string
for i, aRune := range line {
if unicode.IsSpace(aRune) {
if p.HasNextTokenInout(i+1, line) {
tokenIndex++
}
continue
}
aChar := string(aRune)
switch tokenIndex {
case fileInfoPermission:
permission += aChar
case fileInfoOwner:
owner += aChar
case fileInfoGroup:
group += aChar
case fileInfoSize:
if size == "" && !unicode.IsNumber(aRune) {
tokenIndex--
group += " " + aChar
continue
}
size += aChar
case fileInfoDateMonth:
month += aChar
case fileInfoDateDay:
day += aChar
case fileInfoDateHour:
hour += aChar
case fileInfoDateYear:
year += aChar
case fileInfoName:
name += aChar
}
}
if name == "" {
return nil, fmt.Errorf("failed to parse line for fileinfo: %v\n", line)
}
dateTime := year + " " + month + " " + day + " " + hour
layout := toolbox.DateFormatToLayout("yyyy MMM ddd HH:mm:s")
modificationTime, err := time.Parse(layout, dateTime)
if err != nil {
return nil, fmt.Errorf("failed to extract file info from stdout: %v, err: %v", line, err)
}
return p.newObject(parsedURL, name, permission, line, size, modificationTime, isURLDirectory)
}
//extractObjectFromNonIsoBaseTimeCommand extract file storage object from line,
// it expects a file info with iso i.e. -rw-r--r-- 1 awitas awitas 2002 2017-11-04 22:29:33.363458941 +0000 aerospikeciads_aerospike.conf
func (p *Parser) extractObjectFromIsoBasedTimeCommand(parsedURL *url.URL, line string, isURLDirectory bool) (storage.Object, error) {
tokenIndex := 0
if strings.TrimSpace(line) == "" {
return nil, nil
}
var owner, name, permission, group, timezone, date, modTime, size string
line = vtclean.Clean(line, false)
for i, aRune := range line {
if unicode.IsSpace(aRune) {
if p.HasNextTokenInout(i+1, line) {
tokenIndex++
}
continue
}
aChar := string(aRune)
switch tokenIndex {
case fileIsoInfoPermission:
permission += aChar
case fileIsoInfoOwner:
owner += aChar
case fileIsoInfoGroup:
group += aChar
case fileIsoInfoSize:
if size == "" && !unicode.IsNumber(aRune) {
tokenIndex--
group += " " + aChar
continue
}
size += aChar
case fileIsoDate:
date += aChar
case fileIsoTime:
modTime += aChar
case fileIsoTimezone:
timezone += aChar
case fileIsoInfoName:
name += aChar
}
continue
}
timeLen := len(modTime)
if timeLen > 12 {
modTime = string(modTime[:12])
}
dateTime := date + " " + modTime + " " + timezone
layout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss.SSS ZZ")
if len(date+" "+modTime) <= len("yyyy-MM-dd HH:mm:ss") {
layout = toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss ZZ")
}
modificationTime, err := time.Parse(layout, dateTime)
if err != nil {
return nil, fmt.Errorf("failed to extract file info from stdout: %v, err: %v", line, err)
}
return p.newObject(parsedURL, name, permission, line, size, modificationTime, isURLDirectory)
}
toolbox-0.33.2/storage/scp/fileinfo_parser_test.go 0000664 0000000 0000000 00000005104 13741102511 0022246 0 ustar 00root root 0000000 0000000 package scp_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox/storage/scp"
"net/url"
"testing"
)
// -rw-r--r-- 1 awitas wheel 6668 Oct 25 11:41:44 2017 /build/dcm/dcm-server/target/classes/dcm.properties
func Test_ExtractFileInfo(t *testing.T) {
var parserURL, _ = url.Parse("scp://127.0.0.1/")
{
var parser = scp.Parser{IsoTimeStyle: false}
objects, err := parser.Parse(parserURL, "-rw-r--r-- 1 awitas 1742120565 414 Jun 8 14:14:08 2017 f.pub", true)
if assert.Nil(t, err) {
var object = objects[0]
assert.Equal(t, "scp://127.0.0.1/f.pub", object.URL())
assert.Equal(t, int64(1496931248), object.FileInfo().ModTime().Unix())
assert.Equal(t, int64(414), object.FileInfo().Size())
assert.Equal(t, true, object.IsContent())
}
}
{
parserURL, _ = url.Parse("scp://127.0.0.1:22/")
var parser = scp.Parser{IsoTimeStyle: true}
var objects, err = parser.Parse(parserURL, "-rw-r--r-- 1 awitas awitas 2002 2017-11-04 22:29:33.363458941 +0000 aerospikeciads_aerospike.conf", true)
if assert.Nil(t, err) {
var object = objects[0]
assert.Equal(t, "scp://127.0.0.1:22/aerospikeciads_aerospike.conf", object.URL())
assert.Equal(t, int64(1509834573), object.FileInfo().ModTime().Unix())
assert.Equal(t, int64(2002), object.FileInfo().Size())
assert.Equal(t, true, object.IsContent())
}
}
{
parserURL, _ = url.Parse("scp://127.0.0.1:22/")
var parser = scp.Parser{IsoTimeStyle: true}
var objects, err = parser.Parse(parserURL, `-rw------- 1 myusername MYGROUP\\Domain Users 1679 Mar 8 15:27:22 2019 /Users/myusername/.ssh/git_id_rsa`, true)
if assert.Nil(t, err) {
var object = objects[0]
assert.Equal(t, "scp://127.0.0.1:22/Users/myusername/.ssh/git_id_rsa", object.URL())
assert.Equal(t, int64(1552058842), object.FileInfo().ModTime().Unix())
assert.Equal(t, int64(1679), object.FileInfo().Size())
assert.Equal(t, true, object.IsContent())
}
}
{
parserURL, _ = url.Parse("scp://127.0.0.1:22/")
var parser = scp.Parser{IsoTimeStyle: true}
var objects, err = parser.Parse(parserURL, `rwxr-xr-x@ 1 "github.com/viant/toolbox/storage/scp" WORKGROUP\\Domain Users 108143621 Apr 19 15:55:57 2019 /Users/ojoseph/git/test-app/../go-cm/dist/linux/go-cm`, true)
if assert.Nil(t, err) {
var object = objects[0]
assert.Equal(t, "scp://127.0.0.1:22/Users/ojoseph/git/test-app/../go-cm/dist/linux/go-cm", object.URL())
assert.Equal(t, int64(1555689357), object.FileInfo().ModTime().Unix())
assert.Equal(t, int64(108143621), object.FileInfo().Size())
assert.Equal(t, true, object.IsContent())
}
}
}
toolbox-0.33.2/storage/scp/object.go 0000664 0000000 0000000 00000001370 13741102511 0017307 0 ustar 00root root 0000000 0000000 package scp
import (
"github.com/viant/toolbox/storage"
"os"
)
type object struct {
*storage.AbstractObject
source interface{}
permission string
}
//Wrap wraps source storage object
func (i *object) Wrap(source interface{}) {
i.source = source
}
//Unwrap unwraps source storage object into provided target.
func (i *object) Unwrap(target interface{}) error {
if result, ok := target.(**object); ok {
*result = i
}
return nil
}
//newObject creates a new gc storage object
func newStorageObject(URL string, source interface{}, fileInfo os.FileInfo) storage.Object {
abstract := storage.NewAbstractStorageObject(URL, source, fileInfo)
result := &object{
AbstractObject: abstract,
}
result.AbstractObject.Object = result
return result
}
toolbox-0.33.2/storage/scp/provider.go 0000664 0000000 0000000 00000001433 13741102511 0017673 0 ustar 00root root 0000000 0000000 package scp
import (
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/secret"
"github.com/viant/toolbox/storage"
"strings"
)
//ProviderScheme represents scp URL scheme for this provider
const ProviderScheme = "scp"
//SSHProviderScheme represents ssh URL scheme for this provider
const SSHProviderScheme = "ssh"
func init() {
storage.Registry().Registry[ProviderScheme] = serviceProvider
storage.Registry().Registry[SSHProviderScheme] = serviceProvider
}
func serviceProvider(credentials string) (storage.Service, error) {
var config = &cred.Config{}
if strings.TrimSpace(credentials) != "" {
var err error
secrets := secret.New("", false)
config, err = secrets.GetCredentials(credentials)
if err != nil {
return nil, err
}
}
return NewService(config), nil
}
toolbox-0.33.2/storage/scp/service.go 0000664 0000000 0000000 00000020537 13741102511 0017507 0 ustar 00root root 0000000 0000000 package scp
import (
"bytes"
"errors"
"fmt"
"github.com/lunixbochs/vtclean"
"github.com/viant/toolbox"
"github.com/viant/toolbox/cred"
"github.com/viant/toolbox/ssh"
"github.com/viant/toolbox/storage"
"io"
"io/ioutil"
"net/url"
"os"
"path"
"strings"
"sync"
)
const (
defaultSSHPort = 22
verificationSizeThreshold = 1024 * 1024
)
//NoSuchFileOrDirectoryError represents no such file or directory error
var NoSuchFileOrDirectoryError = errors.New("No such file or directory")
const unrecognizedOption = "unrecognized option"
type service struct {
fileService storage.Service
config *cred.Config
services map[string]ssh.Service
multiSessions map[string]ssh.MultiCommandSession
mutex *sync.Mutex
}
func (s *service) runCommand(session ssh.MultiCommandSession, URL string, command string) (string, error) {
s.mutex.Lock()
defer s.mutex.Unlock()
output, _ := session.Run(command, nil, 5000)
var stdout = s.stdout(output)
return stdout, nil
}
func (s *service) stdout(output string) string {
var result = make([]string, 0)
lines := strings.Split(output, "\n")
for _, line := range lines {
result = append(result, vtclean.Clean(line, false))
}
return strings.Join(result, "\n")
}
func (s *service) getMultiSession(parsedURL *url.URL) ssh.MultiCommandSession {
s.mutex.Lock()
defer s.mutex.Unlock()
return s.multiSessions[parsedURL.Host]
}
func (s *service) getService(parsedURL *url.URL) (ssh.Service, error) {
port := toolbox.AsInt(parsedURL.Port())
if port == 0 {
port = 22
}
key := parsedURL.Host
s.mutex.Lock()
defer s.mutex.Unlock()
if service, ok := s.services[key]; ok {
return service, nil
}
service, err := ssh.NewService(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
if err != nil {
return nil, err
}
s.services[key] = service
s.multiSessions[key], err = service.OpenMultiCommandSession(nil)
if err != nil {
return nil, err
}
return service, nil
}
//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
parsedURL, err := url.Parse(URL)
if err != nil {
return nil, err
}
if parsedURL.Host == "127.0.0.1" || parsedURL.Host == "127.0.0.1:22" {
var fileURL = toolbox.FileSchema + parsedURL.Path
return s.fileService.List(fileURL)
}
_, err = s.getService(parsedURL)
if err != nil {
return nil, err
}
commandSession := s.getMultiSession(parsedURL)
canListWithTimeStyle := commandSession.System() != "darwin"
var parser = &Parser{IsoTimeStyle: canListWithTimeStyle}
var URLPath = parsedURL.Path
var result = make([]storage.Object, 0)
var lsCommand = ""
if canListWithTimeStyle {
lsCommand += "ls -dltr --time-style=full-iso " + URLPath
} else {
lsCommand += "ls -dltrT " + URLPath
}
output, _ := s.runCommand(commandSession, URL, lsCommand)
var stdout = vtclean.Clean(string(output), false)
if strings.Contains(stdout, "unrecognized option") {
if canListWithTimeStyle {
lsCommand = "ls -dltr --full-time " + URLPath
output, _ = s.runCommand(commandSession, URL, lsCommand)
stdout = vtclean.Clean(string(output), false)
}
}
if strings.Contains(stdout, unrecognizedOption) {
return nil, fmt.Errorf("unable to list files with: %v, %v", lsCommand, stdout)
}
if strings.Contains(stdout, "No such file or directory") {
return result, NoSuchFileOrDirectoryError
}
objects, err := parser.Parse(parsedURL, stdout, false)
if err != nil {
return nil, err
}
if len(objects) == 1 && objects[0].FileInfo().IsDir() {
output, _ = s.runCommand(commandSession, URL, lsCommand+" "+path.Join(URLPath, "*"))
stdout = vtclean.Clean(string(output), false)
directoryObjects, err := parser.Parse(parsedURL, stdout, true)
if err != nil {
return nil, err
}
if len(directoryObjects) > 0 {
objects = append(objects, directoryObjects...)
}
}
return objects, nil
}
func (s *service) Exists(URL string) (bool, error) {
parsedURL, err := url.Parse(URL)
if err != nil {
return false, err
}
if parsedURL.Host == "127.0.0.1" || parsedURL.Host == "127.0.0.1:22" {
var fileURL = toolbox.FileSchema + parsedURL.Path
return s.fileService.Exists(fileURL)
}
_, err = s.getService(parsedURL)
if err != nil {
return false, err
}
commandSession := s.getMultiSession(parsedURL)
output, _ := s.runCommand(commandSession, URL, "ls -dltr "+parsedURL.Path)
if strings.Contains(string(output), "No such file or directory") {
return false, nil
}
return true, nil
}
func (s *service) StorageObject(URL string) (storage.Object, error) {
objects, err := s.List(URL)
if err != nil {
return nil, err
}
if len(objects) == 0 {
return nil, NoSuchFileOrDirectoryError
}
return objects[0], nil
}
//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.ReadCloser, error) {
if object == nil {
return nil, fmt.Errorf("object was nil")
}
parsedURL, err := url.Parse(object.URL())
if err != nil {
return nil, err
}
if parsedURL.Host == "127.0.0.1" || parsedURL.Host == "127.0.0.1:22" {
var fileURL = toolbox.FileSchema + parsedURL.Path
storageObject, err := s.fileService.StorageObject(fileURL)
if err != nil {
return nil, err
}
return s.fileService.Download(storageObject)
}
port := toolbox.AsInt(parsedURL.Port())
if port == 0 {
port = defaultSSHPort
}
service, err := s.getService(parsedURL)
if err != nil {
return nil, err
}
content, err := service.Download(parsedURL.Path)
if err != nil {
return nil, err
}
return ioutil.NopCloser(bytes.NewReader(content)), nil
}
//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
return s.UploadWithMode(URL, storage.DefaultFileMode, reader)
}
//Upload uploads provided reader content for supplied URL.
func (s *service) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
if mode == 0 {
mode = storage.DefaultFileMode
}
parsedURL, err := url.Parse(URL)
if err != nil {
return err
}
if parsedURL.Host == "127.0.0.1" || parsedURL.Host == "127.0.0.1:22" {
var fileURL = toolbox.FileSchema + parsedURL.Path
return s.fileService.UploadWithMode(fileURL, mode, reader)
}
port := toolbox.AsInt(parsedURL.Port())
if port == 0 {
port = defaultSSHPort
}
//service, err := ssh.NewService(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
service, err := s.getService(parsedURL)
if err != nil {
return err
}
//defer service.Close()
content, err := ioutil.ReadAll(reader)
if err != nil {
return fmt.Errorf("failed to upload - unable read: %v", err)
}
err = service.Upload(parsedURL.Path, mode, content)
if err != nil {
return fmt.Errorf("failed to upload: %v %v", URL, err)
}
return err
}
func (s *service) Register(schema string, service storage.Service) error {
return errors.New("unsupported")
}
func (s *service) Close() error {
for _, service := range s.services {
service.Close()
}
for _, session := range s.multiSessions {
session.Close()
}
return nil
}
//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
parsedURL, err := url.Parse(object.URL())
if err != nil {
return err
}
if parsedURL.Host == "127.0.0.1" || parsedURL.Host == "127.0.0.1:22" {
var fileURL = toolbox.FileSchema + parsedURL.Path
storageObject, err := s.fileService.StorageObject(fileURL)
if err != nil {
return err
}
return s.fileService.Delete(storageObject)
}
port := toolbox.AsInt(parsedURL.Port())
if port == 0 {
port = defaultSSHPort
}
service, err := ssh.NewService(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
if err != nil {
return err
}
//defer service.Close()
session, err := service.NewSession()
if err != nil {
return err
}
defer session.Close()
if parsedURL.Path == "/" {
return fmt.Errorf("invalid removal path: %v", parsedURL.Path)
}
_, err = session.Output("rm -rf " + parsedURL.Path)
return err
}
//DownloadWithURL downloads content for passed in object URL
func (s *service) DownloadWithURL(URL string) (io.ReadCloser, error) {
object, err := s.StorageObject(URL)
if err != nil {
return nil, err
}
return s.Download(object)
}
//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
return &service{
services: make(map[string]ssh.Service),
config: config,
multiSessions: make(map[string]ssh.MultiCommandSession),
mutex: &sync.Mutex{},
fileService: storage.NewFileStorage(),
}
}
toolbox-0.33.2/storage/scp/service_test.go 0000664 0000000 0000000 00000002335 13741102511 0020542 0 ustar 00root root 0000000 0000000 package scp_test
import (
"github.com/stretchr/testify/assert"
"testing"
"github.com/viant/toolbox/storage/scp"
"io/ioutil"
"strings"
)
func TestService_List(t *testing.T) {
//deprecated use viant/afs istead
//service := scp.NewService(nil)
//assert.NotNil(t, service)
//dir, home := path.Split(os.Getenv("HOME"))
//objects, err := service.List("scp://127.0.0.1/" + dir)
//if err == nil {
// return
//}
//assert.Nil(t, err)
//for _, object := range objects {
// if strings.HasSuffix(object.URL(), home) {
// assert.True(t, object.IsFolder())
// }
//}
}
func TestService_Delete(t *testing.T) {
service := scp.NewService(nil)
assert.NotNil(t, service)
err := service.Upload("scp://127.0.0.1//tmp/file.txt", strings.NewReader("this is test"))
assert.Nil(t, err)
objects, err := service.List("scp://127.0.0.1/tmp/")
assert.Nil(t, err)
assert.True(t, len(objects) > 1)
object, err := service.StorageObject("scp://127.0.0.1//tmp/file.txt")
assert.Nil(t, err)
reader, err := service.Download(object)
if err == nil {
defer reader.Close()
content, err := ioutil.ReadAll(reader)
assert.Nil(t, err)
assert.Equal(t, "this is test", string(content))
err = service.Delete(object)
assert.Nil(t, nil)
}
}
toolbox-0.33.2/storage/service.go 0000664 0000000 0000000 00000013154 13741102511 0016717 0 ustar 00root root 0000000 0000000 // Package storage define abstract storage operation
// Deprecated - please use https://github.com/viant/afs API instead
// This package is frozen and no new functionality will be added, and future removal takes place.
package storage
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"strings"
)
var DefaultFileMode os.FileMode = 0755
//Service represents abstract way to accessing local or remote storage
type Service interface {
//List returns a list of object for supplied url
List(URL string) ([]Object, error)
//Exists returns true if resource exists
Exists(URL string) (bool, error)
//Object returns a Object for supplied url
StorageObject(URL string) (Object, error)
//Download returns reader for downloaded storage object
Download(object Object) (io.ReadCloser, error)
//DownloadWithURL returns reader for downloaded URL object
DownloadWithURL(URL string) (io.ReadCloser, error)
//Upload uploads provided reader content for supplied storage object.
Upload(URL string, reader io.Reader) error
//Upload uploads provided reader content for supplied storage object.
UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error
//Delete removes passed in storage object
Delete(object Object) error
//Register register schema with provided service
Register(schema string, service Service) error
//Closes storage service
Close() error
}
type storageService struct {
registry map[string]Service
}
func (s *storageService) getServiceForSchema(URL string) (Service, error) {
parsedUrl, err := url.Parse(URL)
if err != nil {
return nil, err
}
if result, found := s.registry[parsedUrl.Scheme]; found {
return result, nil
}
return nil, fmt.Errorf("failed to lookup url schema %v in %v", parsedUrl.Scheme, URL)
}
//List lists all object for passed in URL
func (s *storageService) List(URL string) ([]Object, error) {
service, err := s.getServiceForSchema(URL)
if err != nil {
return nil, err
}
return service.List(URL)
}
//Exists returns true if resource exists
func (s *storageService) Exists(URL string) (bool, error) {
service, err := s.getServiceForSchema(URL)
if err != nil {
return false, err
}
return service.Exists(URL)
}
//StorageObject returns storage object for provided URL
func (s *storageService) StorageObject(URL string) (Object, error) {
service, err := s.getServiceForSchema(URL)
if err != nil {
return nil, err
}
return service.StorageObject(URL)
}
//Download downloads content for passed in object
func (s *storageService) Download(object Object) (io.ReadCloser, error) {
service, err := s.getServiceForSchema(object.URL())
if err != nil {
return nil, err
}
return service.Download(object)
}
//DownloadWithURL downloads content for passed in object URL
func (s *storageService) DownloadWithURL(URL string) (io.ReadCloser, error) {
object, err := s.StorageObject(URL)
if err != nil {
return nil, err
}
return s.Download(object)
}
//Uploads content for passed in URL
func (s *storageService) Upload(URL string, reader io.Reader) error {
service, err := s.getServiceForSchema(URL)
if err != nil {
return err
}
return service.UploadWithMode(URL, 0644, reader)
}
//Uploads content for passed in URL
func (s *storageService) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
service, err := s.getServiceForSchema(URL)
if err != nil {
return err
}
return service.UploadWithMode(URL, mode, reader)
}
//Delete remove storage object
func (s *storageService) Delete(object Object) error {
service, err := s.getServiceForSchema(object.URL())
if err != nil {
return err
}
return service.Delete(object)
}
//Close closes resources
func (s *storageService) Close() error {
for _, service := range s.registry {
err := service.Close()
if err != nil {
return err
}
}
return nil
}
//Register register storage schema
func (s *storageService) Register(schema string, service Service) error {
s.registry[schema] = service
return nil
}
//NewService creates a new storage service
func NewService() Service {
var result = &storageService{
registry: make(map[string]Service),
}
_ = result.Register("file", &fileStorageService{})
_ = result.Register("mem", NewMemoryService())
return result
}
//NewServiceForURL creates a new storage service for provided URL scheme and optional credential file
func NewServiceForURL(URL, credentials string) (Service, error) {
parsedURL, err := url.Parse(URL)
if err != nil {
return nil, err
}
service := NewService()
provider := Registry().Get(parsedURL.Scheme)
if provider != nil {
if len(credentials) > 0 {
credentials = strings.Replace(credentials, "${env.HOME}", os.Getenv("HOME"), 1)
if strings.HasPrefix(credentials, "~") {
credentials = strings.Replace(credentials, "~", os.Getenv("HOME"), 1)
}
}
serviceForScheme, err := provider(credentials)
if err != nil {
return nil, fmt.Errorf("failed lookup service for %v: %v", parsedURL.Scheme, err)
}
err = service.Register(parsedURL.Scheme, serviceForScheme)
if err != nil {
return nil, err
}
} else if parsedURL.Scheme != "file" {
return nil, fmt.Errorf("unsupported scheme %v", URL)
}
return service, nil
}
//Download returns a download reader for supplied URL
func Download(service Service, URL string) (io.ReadCloser, error) {
object, err := service.StorageObject(URL)
if err != nil {
return nil, err
}
return service.Download(object)
}
//DownloadText returns a text for supplied URL
func DownloadText(service Service, URL string) (string, error) {
reader, err := Download(service, URL)
if err != nil {
return "", err
}
defer reader.Close()
content, err := ioutil.ReadAll(reader)
if err != nil {
return "", err
}
return string(content), nil
}
toolbox-0.33.2/storage/service_test.go 0000664 0000000 0000000 00000004273 13741102511 0017760 0 ustar 00root root 0000000 0000000 package storage_test
import (
"bytes"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
_ "github.com/viant/toolbox/storage/scp"
"io/ioutil"
"os"
"os/exec"
"path"
"strings"
"testing"
)
func TestStorageService_List(t *testing.T) {
service := storage.NewService()
assert.NotNil(t, service)
fileName, _, _ := toolbox.CallerInfo(2)
parent, _ := path.Split(fileName)
baseUrl := "file://" + parent + "/test"
if toolbox.FileExists(parent + "/test/file3.txt") {
os.Remove(parent + "/test/file3.txt")
}
defer os.Remove(parent + "/test/file3.txt")
objects, err := service.List(baseUrl)
assert.Nil(t, err)
assert.True(t, len(objects) >= 5)
var objectByUrl = make(map[string]storage.Object)
for _, object := range objects {
objectByUrl[object.URL()] = object
}
assert.NotNil(t, objectByUrl[baseUrl+"/dir"])
assert.NotNil(t, objectByUrl[baseUrl+"/file1.txt"])
assert.NotNil(t, objectByUrl[baseUrl+"/file2.txt"])
assert.True(t, objectByUrl[baseUrl+"/dir"].IsFolder())
assert.True(t, objectByUrl[baseUrl+"/file2.txt"].IsContent())
{
reader, err := service.Download(objectByUrl[baseUrl+"/file2.txt"])
if assert.Nil(t, err) {
defer reader.Close()
content, err := ioutil.ReadAll(reader)
assert.Nil(t, err)
assert.Equal(t, "line1\nline2", string(content))
}
}
var newFileUrl = baseUrl + "/file3.txt"
err = service.Upload(baseUrl+"/file3.txt", bytes.NewReader([]byte("abc")))
assert.Nil(t, err)
exists, err := service.Exists(baseUrl + "/file3.txt")
assert.Nil(t, err)
assert.True(t, exists)
{
object, err := service.StorageObject(newFileUrl)
assert.Nil(t, err)
reader, err := service.Download(object)
if assert.Nil(t, err) {
defer reader.Close()
content, err := ioutil.ReadAll(reader)
assert.Nil(t, err)
assert.Equal(t, "abc", string(content))
}
}
}
func TestUpload(t *testing.T) {
var path = "/tmp/local/test.txt"
toolbox.RemoveFileIfExist(path)
exec.Command("rmdir /tmp/local").CombinedOutput()
var destination = "scp://127.0.0.1/" + path
service, err := storage.NewServiceForURL(destination, "")
assert.Nil(t, err)
err = service.Upload(destination, strings.NewReader("abc"))
assert.Nil(t, err)
}
toolbox-0.33.2/storage/test/ 0000775 0000000 0000000 00000000000 13741102511 0015703 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/test/dir/ 0000775 0000000 0000000 00000000000 13741102511 0016461 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/test/dir/file.json 0000664 0000000 0000000 00000000040 13741102511 0020265 0 ustar 00root root 0000000 0000000 [{
"id":1,
"name":"name1"
}] toolbox-0.33.2/storage/test/file1.txt 0000664 0000000 0000000 00000000721 13741102511 0017444 0 ustar 00root root 0000000 0000000 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ultricies nisl id quam laoreet porttitor quis eu nulla. Sed pellentesque sit amet turpis a accumsan. Maecenas id odio ac dolor egestas iaculis. Maecenas non velit gravida, egestas nunc eget, dictum nisi. Quisque lorem ex, ullamcorper ac justo ut, vehicula sollicitudin risus. Duis imperdiet facilisis gravida. In hac habitasse platea dictumst. Ut sollicitudin lectus tortor. Fusce eu turpis justo. toolbox-0.33.2/storage/test/file2.txt 0000664 0000000 0000000 00000000013 13741102511 0017437 0 ustar 00root root 0000000 0000000 line1
line2 toolbox-0.33.2/storage/test/mock_s3.json 0000664 0000000 0000000 00000000117 13741102511 0020133 0 ustar 00root root 0000000 0000000 {
"Region":"us-east-1",
"Key":"MOCKKEY",
"Secret":"thisisasecretstring"
} toolbox-0.33.2/storage/test/source/ 0000775 0000000 0000000 00000000000 13741102511 0017203 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/test/source/dir/ 0000775 0000000 0000000 00000000000 13741102511 0017761 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/test/source/dir/file.json 0000664 0000000 0000000 00000000040 13741102511 0021565 0 ustar 00root root 0000000 0000000 [{
"id":1,
"name":"name1"
}] toolbox-0.33.2/storage/test/source/dir2/ 0000775 0000000 0000000 00000000000 13741102511 0020043 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/test/source/dir2/subdir/ 0000775 0000000 0000000 00000000000 13741102511 0021333 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/storage/test/source/dir2/subdir/file1.txt 0000664 0000000 0000000 00000000721 13741102511 0023074 0 ustar 00root root 0000000 0000000 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ultricies nisl id quam laoreet porttitor quis eu nulla. Sed pellentesque sit amet turpis a accumsan. Maecenas id odio ac dolor egestas iaculis. Maecenas non velit gravida, egestas nunc eget, dictum nisi. Quisque lorem ex, ullamcorper ac justo ut, vehicula sollicitudin risus. Duis imperdiet facilisis gravida. In hac habitasse platea dictumst. Ut sollicitudin lectus tortor. Fusce eu turpis justo. toolbox-0.33.2/storage/test/source/file1.txt 0000664 0000000 0000000 00000000721 13741102511 0020744 0 ustar 00root root 0000000 0000000 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ultricies nisl id quam laoreet porttitor quis eu nulla. Sed pellentesque sit amet turpis a accumsan. Maecenas id odio ac dolor egestas iaculis. Maecenas non velit gravida, egestas nunc eget, dictum nisi. Quisque lorem ex, ullamcorper ac justo ut, vehicula sollicitudin risus. Duis imperdiet facilisis gravida. In hac habitasse platea dictumst. Ut sollicitudin lectus tortor. Fusce eu turpis justo. toolbox-0.33.2/storage/test/source/file2.txt 0000664 0000000 0000000 00000000013 13741102511 0020737 0 ustar 00root root 0000000 0000000 line1
line2 toolbox-0.33.2/struct_helper.go 0000664 0000000 0000000 00000037162 13741102511 0016503 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"github.com/go-errors/errors"
"reflect"
"strings"
)
const (
fieldNameKey = "fieldName"
anonymousKey = "anonymous"
fieldIndexKey = "fieldIndex"
defaultKey = "default"
)
var columnMapping = []string{"column", "dateLayout", "dateFormat", "autoincrement", "primaryKey", "sequence", "valueMap", defaultKey, anonymousKey}
//ScanStructFunc scan supplied struct methods
func ScanStructMethods(structOrItsType interface{}, depth int, handler func(method reflect.Method) error) error {
var scanned = make(map[reflect.Type]bool)
return scanStructMethods(structOrItsType, scanned, depth, handler)
}
func scanStructMethods(structOrItsType interface{}, scanned map[reflect.Type]bool, depth int, handler func(method reflect.Method) error) error {
if depth < 0 {
return nil
}
structValue, err := TryDiscoverValueByKind(reflect.ValueOf(structOrItsType), reflect.Struct)
if err != nil {
structValue := reflect.ValueOf(structOrItsType)
if !(structValue.Kind() == reflect.Interface) {
return err
}
}
structType := structValue.Type()
if _, hasScan := scanned[structType]; hasScan {
return nil
}
scanned[structType] = true
for i := 0; i < structValue.NumField(); i++ {
fieldType := structType.Field(i)
if isExported := fieldType.PkgPath == ""; !isExported {
continue
}
if !fieldType.Anonymous {
continue
}
if !IsStruct(fieldType) {
continue
}
if fieldStructType, err := TryDiscoverTypeByKind(fieldType, reflect.Struct); err == nil {
fieldStruct := reflect.New(fieldStructType).Interface()
if err = scanStructMethods(fieldStruct, scanned, depth-1, handler); err != nil {
return err
}
}
}
structPtr, err := TryDiscoverValueByKind(reflect.ValueOf(structOrItsType), reflect.Ptr)
if err != nil {
return err
}
structTypePtr := structPtr.Type()
for i := 0; i < structTypePtr.NumMethod(); i++ {
method := structTypePtr.Method(i)
if isExported := method.PkgPath == ""; !isExported {
continue
}
if err := handler(method); err != nil {
return err
}
}
return nil
}
//StructField represents a struct field
type StructField struct {
Owner reflect.Value
Value reflect.Value
Type reflect.StructField
}
var onUnexportedHandler = IgnoreUnexportedFields
//UnexportedFieldHandler represents unexported field handler
type UnexportedFieldHandler func(structField *StructField) bool
//Handler ignoring unexported fields
func IgnoreUnexportedFields(structField *StructField) bool {
return false
}
func SetUnexportedFieldHandler(handler UnexportedFieldHandler) error {
if handler == nil {
return errors.New("handler was nil")
}
onUnexportedHandler = handler
return nil
}
//ProcessStruct reads passed in struct fields and values to pass it to provided handler
func ProcessStruct(aStruct interface{}, handler func(fieldType reflect.StructField, field reflect.Value) error) error {
structValue, err := TryDiscoverValueByKind(reflect.ValueOf(aStruct), reflect.Struct)
if err != nil {
return err
}
structType := structValue.Type()
var fields = make(map[string]*StructField)
for i := 0; i < structType.NumField(); i++ {
fieldType := structType.Field(i)
if !fieldType.Anonymous {
continue
}
field := structValue.Field(i)
if !IsStruct(field) {
fields[fieldType.Name] = &StructField{Type: fieldType, Value: field, Owner: structValue}
continue
}
var aStruct interface{}
if fieldType.Type.Kind() == reflect.Ptr {
if field.IsNil() {
if !field.CanSet() {
continue
}
structValue.Field(i).Set(reflect.New(fieldType.Type.Elem()))
}
aStruct = field.Interface()
} else {
if field.CanAddr() {
aStruct = field.Addr().Interface()
} else if field.CanInterface() {
aStruct = field.Interface()
} else {
continue
}
}
if err := ProcessStruct(aStruct, func(fieldType reflect.StructField, field reflect.Value) error {
structField := &StructField{Type: fieldType, Value: field, Owner: field}
if field.CanAddr() {
structField.Owner = field.Addr()
}
fields[fieldType.Name] = structField
return nil
}); err != nil {
return err
}
}
for i := 0; i < structType.NumField(); i++ {
fieldType := structType.Field(i)
if fieldType.Anonymous {
continue
}
field := structValue.Field(i)
structField := &StructField{Owner: structValue, Type: fieldType, Value: field}
if isExported := fieldType.PkgPath == ""; !isExported {
if !onUnexportedHandler(structField) {
continue
}
}
fields[fieldType.Name] = &StructField{Owner: structValue, Type: fieldType, Value: field}
}
for _, field := range fields {
if err := handler(field.Type, field.Value); err != nil {
return err
}
}
return nil
}
//BuildTagMapping builds map keyed by mappedKeyTag tag value, and value is another map of keys where tag name is presents in the tags parameter.
func BuildTagMapping(structTemplatePointer interface{}, mappedKeyTag string, resultExclusionTag string, inheritKeyFromField bool, convertKeyToLowerCase bool, tags []string) map[string](map[string]string) {
reflectStructType := DiscoverTypeByKind(structTemplatePointer, reflect.Struct)
var result = make(map[string]map[string]string)
var anonymousMappings = make(map[string]map[string]string)
for i := 0; i < reflectStructType.NumField(); i++ {
var field reflect.StructField
field = reflectStructType.Field(i)
key := getTagValues(field, mappedKeyTag)
if field.Anonymous && key == "" {
var anonymousType = DereferenceType(field.Type)
if anonymousType.Kind() == reflect.Struct {
anonymousMapping := BuildTagMapping(reflect.New(anonymousType).Interface(), mappedKeyTag, resultExclusionTag, inheritKeyFromField, convertKeyToLowerCase, tags)
for k, v := range anonymousMapping {
anonymousMappings[k] = v
anonymousMappings[k][anonymousKey] = "true"
anonymousMappings[k][fieldIndexKey] = AsString(i)
}
}
continue
}
isTransient := strings.EqualFold(field.Tag.Get(resultExclusionTag), "true")
if isTransient {
continue
}
if key == "" {
if !inheritKeyFromField {
continue
}
key = field.Name
}
if convertKeyToLowerCase {
key = strings.ToLower(key)
}
result[key] = make(map[string]string)
for _, tag := range tags {
tagValue := field.Tag.Get(tag)
if len(tagValue) > 0 {
result[key][tag] = tagValue
}
}
result[key][fieldNameKey] = field.Name
}
for k, v := range anonymousMappings {
if _, has := result[k]; !has {
result[k] = v
}
}
return result
}
func getTagValues(field reflect.StructField, mappedKeyTag string) string {
key := field.Tag.Get(mappedKeyTag)
key = strings.Split(key, ",")[0]
if mappedKeyTag == fieldNameKey {
key = field.Name
}
return key
}
//NewFieldSettingByKey reads field's tags and returns them indexed by passed in key, fieldName is always part of the resulting map unless filed has "transient" tag.
func NewFieldSettingByKey(aStruct interface{}, key string) map[string](map[string]string) {
return BuildTagMapping(aStruct, key, "transient", true, true, columnMapping)
}
func setEmptyMap(source reflect.Value, dataTypes map[string]bool) {
if !source.CanSet() {
return
}
mapType := source.Type()
mapPointer := reflect.New(mapType)
mapValueType := mapType.Elem()
mapKeyType := mapType.Key()
newMap := mapPointer.Elem()
newMap.Set(reflect.MakeMap(mapType))
targetMapKeyPointer := reflect.New(mapKeyType)
targetMapValuePointer := reflect.New(mapValueType)
var elementKey = targetMapKeyPointer.Elem()
var elementValue = targetMapValuePointer.Elem()
if elementValue.Kind() == reflect.Ptr && elementValue.IsNil() {
component := reflect.New(elementValue.Type().Elem())
elementValue.Set(component)
}
if elementKey.Type() != mapKeyType {
if elementKey.Type().AssignableTo(mapKeyType) {
elementKey = elementKey.Convert(mapKeyType)
}
}
if DereferenceType(elementValue.Type()).Kind() == reflect.Struct {
initStruct(elementValue.Interface(), dataTypes)
}
newMap.SetMapIndex(elementKey, elementValue)
var elem = mapPointer.Elem()
source.Set(elem)
}
func createEmptySlice(source reflect.Value, dataTypes map[string]bool) {
sliceType := DiscoverTypeByKind(source.Type(), reflect.Slice)
if !source.CanSet() {
return
}
slicePointer := reflect.New(sliceType)
slice := slicePointer.Elem()
componentType := DiscoverComponentType(sliceType)
var targetComponentPointer = reflect.New(componentType)
var targetComponent = targetComponentPointer.Elem()
if DereferenceType(componentType).Kind() == reflect.Struct {
componentType := targetComponent.Type()
isPointer := componentType.Kind() == reflect.Ptr
if isPointer {
componentType = componentType.Elem()
}
structElement := reflect.New(componentType)
initStruct(structElement.Interface(), dataTypes)
if isPointer {
targetComponentPointer.Elem().Set(structElement)
} else {
targetComponentPointer.Elem().Set(structElement.Elem())
}
initStruct(targetComponentPointer.Elem().Interface(), dataTypes)
}
slice.Set(reflect.Append(slice, targetComponentPointer.Elem()))
source.Set(slicePointer.Elem())
}
//InitStruct initialise any struct pointer to empty struct
func InitStruct(source interface{}) {
var dataTypes = make(map[string]bool)
if source == nil {
return
}
initStruct(source, dataTypes)
}
func initStruct(source interface{}, dataTypes map[string]bool) {
if source == nil {
return
}
if !IsStruct(source) {
return
}
var key = DereferenceType(source).Name()
if _, has := dataTypes[key]; has {
return
}
dataTypes[key] = true
sourceValue, ok := source.(reflect.Value)
if !ok {
sourceValue = reflect.ValueOf(source)
}
if sourceValue.Type().Kind() == reflect.Ptr {
elem := sourceValue.Elem()
if elem.Kind() == reflect.Ptr && elem.IsNil() {
return
}
if !sourceValue.Elem().IsValid() {
return
}
}
_ = ProcessStruct(source, func(fieldType reflect.StructField, fieldValue reflect.Value) error {
if !fieldValue.CanInterface() {
return nil
}
if fieldValue.Kind() == reflect.String && fieldValue.CanSet() {
fieldValue.SetString(" ")
return nil
}
if fieldType.Type.Kind() == reflect.Map {
setEmptyMap(fieldValue, dataTypes)
return nil
}
if fieldType.Type.Kind() == reflect.Slice {
createEmptySlice(fieldValue, dataTypes)
return nil
}
if fieldType.Type.Kind() != reflect.Ptr {
return nil
}
if DereferenceType(fieldType).Kind() == reflect.Struct {
if !fieldValue.CanSet() {
return nil
}
if fieldValue.Type().Kind() == reflect.Ptr {
fieldStruct := reflect.New(fieldValue.Type().Elem())
if reflect.TypeOf(source) != fieldStruct.Type() {
initStruct(fieldStruct.Interface(), dataTypes)
}
fieldValue.Set(fieldStruct)
}
}
return nil
})
}
//StructFieldMeta represents struct field meta
type StructFieldMeta struct {
Name string `json:"name,omitempty"`
Type string `json:"type,omitempty"`
Required bool `json:"required,"`
Description string `json:"description,omitempty"`
}
//StructMeta represents struct meta details
type StructMeta struct {
Type string
rawType reflect.Type `json:"-"`
Fields []*StructFieldMeta `json:"fields,omitempty"`
Dependencies []*StructMeta `json:"dependencies,omitempty"`
}
func (m *StructMeta) Message() map[string]interface{} {
var result = make(map[string]interface{})
var deps = make(map[string]*StructMeta)
for _, dep := range m.Dependencies {
deps[dep.Type] = dep
}
for _, field := range m.Fields {
if dep, ok := deps[field.Type]; ok {
result[field.Name] = dep.Message()
continue
}
result[field.Name] = ""
}
return result
}
//StructMetaFilter
type StructMetaFilter func(field reflect.StructField) bool
func DefaultStructMetaFilter(ield reflect.StructField) bool {
return true
}
var structMetaFilter StructMetaFilter = DefaultStructMetaFilter
//SetStructMetaFilter sets struct meta filter
func SetStructMetaFilter(filter StructMetaFilter) error {
if filter == nil {
return errors.New("filter was nil")
}
structMetaFilter = filter
return nil
}
//GetStructMeta returns struct meta
func GetStructMeta(source interface{}) *StructMeta {
var result = &StructMeta{}
var trackedTypes = make(map[string]bool)
getStructMeta(source, result, trackedTypes)
return result
}
//InitStruct initialise any struct pointer to empty struct
func getStructMeta(source interface{}, meta *StructMeta, trackedTypes map[string]bool) bool {
if source == nil {
return false
}
var structType = fmt.Sprintf("%T", source)
if _, has := trackedTypes[structType]; has {
return false
}
meta.Type = structType
meta.Fields = make([]*StructFieldMeta, 0)
meta.Dependencies = make([]*StructMeta, 0)
sourceValue := reflect.ValueOf(source)
if sourceValue.Kind() == reflect.Ptr {
elem := sourceValue.Elem()
if elem.Kind() == reflect.Ptr && elem.IsNil() {
return false
}
if !sourceValue.Elem().IsValid() {
source = reflect.New(sourceValue.Type().Elem()).Interface()
}
}
meta.rawType = sourceValue.Type()
trackedTypes[structType] = true
_ = ProcessStruct(source, func(fieldType reflect.StructField, field reflect.Value) error {
if !structMetaFilter(fieldType) {
return nil
}
if isExported := fieldType.PkgPath == ""; !isExported {
structField := &StructField{
Owner: reflect.ValueOf(source),
Type: fieldType,
Value: field,
}
if !onUnexportedHandler(structField) {
return nil
}
field = structField.Value
}
if isJSONSkippable(string(fieldType.Tag)) {
return nil
}
fieldMeta := &StructFieldMeta{}
fieldMeta.Name = fieldType.Name
fieldMeta.Type = fieldType.Type.Name()
meta.Fields = append(meta.Fields, fieldMeta)
if value, ok := fieldType.Tag.Lookup("required"); ok {
fieldMeta.Required = AsBoolean(value)
}
if value, ok := fieldType.Tag.Lookup("description"); ok {
fieldMeta.Description = value
}
var value = field.Interface()
if value == nil {
return nil
}
fieldMeta.Type = fmt.Sprintf("%T", value)
if fieldType.PkgPath != "" {
fieldMeta.Type = strings.Replace(fieldMeta.Type, "*", "", 1)
}
if IsStruct(value) {
var fieldStruct = &StructMeta{}
switch field.Kind() {
case reflect.Ptr:
var fieldValue interface{}
if field.IsNil() {
fieldValue = reflect.New(field.Type().Elem()).Interface()
} else {
fieldValue = field.Elem().Interface()
}
if getStructMeta(fieldValue, fieldStruct, trackedTypes) {
meta.Dependencies = append(meta.Dependencies, fieldStruct)
}
case reflect.Struct:
if field.CanInterface() {
if getStructMeta(field.Interface(), fieldStruct, trackedTypes) {
meta.Dependencies = append(meta.Dependencies, fieldStruct)
}
}
}
return nil
}
if IsMap(value) {
var aMap = AsMap(field.Interface())
var mapValue interface{}
for _, mapValue = range aMap {
break
}
if mapValue != nil && IsStruct(mapValue) {
var fieldStruct = &StructMeta{}
if getStructMeta(mapValue, fieldStruct, trackedTypes) {
meta.Dependencies = append(meta.Dependencies, fieldStruct)
}
}
return nil
}
if IsSlice(value) {
var aSlice = AsSlice(field.Interface())
if len(aSlice) > 0 {
if aSlice[0] != nil && IsStruct(aSlice[0]) {
var fieldStruct = &StructMeta{}
if getStructMeta(aSlice[0], fieldStruct, trackedTypes) {
meta.Dependencies = append(meta.Dependencies, fieldStruct)
}
}
}
return nil
}
return nil
})
return true
}
func isJSONSkippable(tag string) bool {
return strings.Contains(tag, "json:\"-")
}
//StructFields by name sorter
type StructFields []*StructField
// Len is part of sort.Interface.
func (s StructFields) Len() int {
return len(s)
}
// Swap is part of sort.Interface.
func (s StructFields) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
// Less is part of sort.Interface.
func (s StructFields) Less(i, j int) bool {
return s[i].Type.Name < s[j].Type.Name
}
toolbox-0.33.2/struct_helper_test.go 0000664 0000000 0000000 00000011454 13741102511 0017536 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"reflect"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
)
func TestProcessStruct(t *testing.T) {
type Super struct {
Parent int
}
type User struct {
*Super
Name string `column:"name"`
DateOfBirth time.Time `column:"date" dateFormat:"2006-01-02 15:04:05.000000"`
Id int `autogenrated:"true"`
prv int
Other string `transient:"true"`
}
user := User{Id: 1, Other: "!@#", Name: "foo", Super: &Super{12}}
var userMap = make(map[string]interface{})
err := toolbox.ProcessStruct(&user, func(fieldType reflect.StructField, field reflect.Value) error {
value := field.Interface()
userMap[fieldType.Name] = value
return nil
})
assert.Nil(t, err)
assert.Equal(t, 5, len(userMap))
assert.Equal(t, 12, userMap["Parent"])
assert.Equal(t, 1, userMap["Id"])
assert.Equal(t, "!@#", userMap["Other"])
}
func TestBuildTagMapping(t *testing.T) {
type User struct {
Name string `column:"name"`
DateOfBirth time.Time `column:"date" dateFormat:"2006-01-02 15:04:05.000000"`
Id int `autogenrated:"true"`
Other string `transient:"true"`
}
{
tags := []string{"column", "autogenrated"}
result := toolbox.BuildTagMapping((*User)(nil), "column", "transient", true, true, tags)
{
actual := len(result)
expected := 3
assert.Equal(t, actual, expected, "Extract mapping count")
}
{
actual, _ := result["name"]["fieldName"]
expected := "Name"
assert.Equal(t, actual, expected, "Extract name mapping")
}
{
actual, _ := result["id"]["autogenrated"]
expected := "true"
assert.Equal(t, actual, expected, "Extract id flaged as autogenerated")
}
}
{
tags := []string{"column", "autogenrated"}
result := toolbox.BuildTagMapping((*User)(nil), "fieldName", "transient", true, false, tags)
actual, _ := result["Name"]["fieldName"]
expected := "Name"
assert.Equal(t, actual, expected, "Extract name mapping")
}
{
tags := []string{"column", "autogenrated"}
result := toolbox.BuildTagMapping((*User)(nil), "column", "transient", false, false, tags)
{
actual := len(result)
expected := 2
assert.Equal(t, actual, expected, "Extract mapping count")
}
}
type User2 struct {
Name string `json:"name" column:"name"`
DateOfBirth time.Time `json:"date" column:"date" dateFormat:"2006-01-02 15:04:05.000000"`
Id int `json:"id" autogenrated:"true"`
Other string `json:"other" transient:"true"`
}
{
tags := []string{"column", "autogenrated"}
result := toolbox.BuildTagMapping((*User)(nil), "column", "transient", true, true, tags)
{
actual := len(result)
expected := 3
assert.Equal(t, actual, expected, "Extract mapping count")
}
{
actual, _ := result["name"]["fieldName"]
expected := "Name"
assert.Equal(t, actual, expected, "Extract name mapping")
}
{
actual, _ := result["id"]["autogenrated"]
expected := "true"
assert.Equal(t, actual, expected, "Extract id flaged as autogenerated")
}
}
}
func TestBuildEmbededStructTagMapping(t *testing.T) {
type Super struct {
Id int `autogenrated:"true"`
Name string `column:"name"`
}
type User struct {
*Super
DateOfBirth time.Time `column:"date" dateFormat:"2006-01-02 15:04:05.000000"`
Other string `transient:"true"`
}
tags := []string{"column", "autogenrated"}
result := toolbox.BuildTagMapping((*User)(nil), "column", "transient", true, true, tags)
{
actual := len(result)
expected := 3
assert.Equal(t, actual, expected, "Extract mapping count")
}
{
actual, _ := result["name"]["fieldName"]
expected := "Name"
assert.Equal(t, actual, expected, "Extract name mapping")
}
{
actual, _ := result["id"]["autogenrated"]
expected := "true"
assert.Equal(t, actual, expected, "Extract id flaged as autogenerated")
}
}
type Type4 struct {
Id int
}
type Type3 struct {
Name map[string]string
Type4 map[string]*Type4
}
type Type2 struct {
F1 int
F3 *Type3
}
type Type1 struct {
F1 int
F2 *Type2
F3 []interface{}
F4 map[string]interface{}
F5 []*Type3
}
type SuperType1 struct {
*Type1
}
type SuperType2 struct {
Type1
}
func Test_InitStruct(t *testing.T) {
{
var t1 = &Type1{}
toolbox.InitStruct(t1)
assert.NotNil(t, t1.F2)
assert.NotNil(t, t1.F3)
assert.NotNil(t, t1.F4)
assert.NotNil(t, t1.F5)
}
{
var t1 = &SuperType1{}
toolbox.InitStruct(t1)
assert.NotNil(t, t1.F2)
assert.NotNil(t, t1.F3)
assert.NotNil(t, t1.F4)
assert.NotNil(t, t1.F5)
}
{
var t1 = &SuperType2{}
toolbox.InitStruct(t1)
assert.NotNil(t, t1.F2)
assert.NotNil(t, t1.F3)
assert.NotNil(t, t1.F4)
assert.NotNil(t, t1.F5)
}
}
func Test_GetStructMeta(t *testing.T) {
var t1 = &Type1{}
toolbox.InitStruct(t1)
meta := toolbox.GetStructMeta(t1)
assert.NotNil(t, meta)
}
toolbox-0.33.2/test/ 0000775 0000000 0000000 00000000000 13741102511 0014237 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/test/config.json 0000664 0000000 0000000 00000000050 13741102511 0016372 0 ustar 00root root 0000000 0000000 {
"Attr1":"value1",
"Attr2":"value2"
} toolbox-0.33.2/test/corrupted_config.json 0000664 0000000 0000000 00000000061 13741102511 0020463 0 ustar 00root root 0000000 0000000 {
"Attr1":"value1",
"Attr2":"value2"
"attr";
} toolbox-0.33.2/test/fileset_info/ 0000775 0000000 0000000 00000000000 13741102511 0016705 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/test/fileset_info/user.go 0000664 0000000 0000000 00000001501 13741102511 0020207 0 ustar 00root root 0000000 0000000 package fileset_info
import (
"fmt"
"time"
)
type Addresses []Address
type Ints []int
type Z string
type Country struct {
Code string
Name string
}
//Address represents a test struct
type Address struct {
Country
City string
}
type A Address
//User represents a test struct
type User struct { //my comments
///abc comment
ID *int // comment1 type
Name string
DateOfBirth time.Time `foo="bar"`
Address Address
AddressPointer *Address
Addresses Addresses
Ints []int
Ints2 Ints
M map[string][]string
C chan *bool
Appointments []time.Time
}
//Test represents a test method
func (u User) Test() {
fmt.Printf("Abc %v", u)
}
//Test1 represents a test method
func (u User) Test1() bool {
fmt.Printf("Abc %v", u)
return false
}
toolbox-0.33.2/test/fileset_info/user_func.go 0000664 0000000 0000000 00000000170 13741102511 0021223 0 ustar 00root root 0000000 0000000 package fileset_info
import "fmt"
//Test represents a test method
func (u *User) Test2() {
fmt.Printf("Abc %v", u)
}
toolbox-0.33.2/text.go 0000664 0000000 0000000 00000007625 13741102511 0014605 0 ustar 00root root 0000000 0000000 package toolbox
import (
"bufio"
"io"
"unicode"
)
//IsASCIIText return true if supplied string does not have binary data
func IsASCIIText(candidate string) bool {
for _, r := range candidate {
if r == '\n' || r == '\r' || r == '\t' {
continue
}
if r > unicode.MaxASCII || !unicode.IsPrint(r) || r == '`' {
return false
}
}
return true
}
//IsPrintText return true if all candidate characters are printable (unicode.IsPrintText)
func IsPrintText(candidate string) bool {
for _, r := range candidate {
if !unicode.IsPrint(r) {
if r == '\n' || r == '\r' || r == '\t' || r == '`' {
continue
}
return false
}
}
return true
}
//TerminatedSplitN split supplied text into n fragmentCount, each terminated with supplied terminator
func TerminatedSplitN(text string, fragmentCount int, terminator string) []string {
var result = make([]string, 0)
if fragmentCount == 0 {
fragmentCount = 1
}
fragmentSize := len(text) / fragmentCount
lowerBound := 0
for i := fragmentSize - 1; i < len(text); i++ {
isLast := i+1 == len(text)
isAtLeastOfFragmentSize := i-lowerBound >= fragmentSize
isNewLine := string(text[i:i+len(terminator)]) == terminator
if (isAtLeastOfFragmentSize && isNewLine) || isLast {
result = append(result, string(text[lowerBound:i+1]))
lowerBound = i + 1
}
}
return result
}
//SplitTextStream divides reader supplied text by number of specified line
func SplitTextStream(reader io.Reader, writerProvider func() io.WriteCloser, elementCount int) error {
scanner := bufio.NewScanner(reader)
var writer io.WriteCloser
counter := 0
var err error
if elementCount == 0 {
elementCount = 1
}
for scanner.Scan() {
if writer == nil {
writer = writerProvider()
}
data := scanner.Bytes()
if err = scanner.Err(); err != nil {
return err
}
if counter > 0 {
if _, err = writer.Write([]byte{'\n'}); err != nil {
return err
}
}
if _, err = writer.Write(data); err != nil {
return err
}
counter++
if counter == elementCount {
if err := writer.Close(); err != nil {
return err
}
counter = 0
writer = nil
}
}
if writer != nil {
return writer.Close()
}
return nil
}
const (
CaseUpper = iota
CaseLower
CaseUpperCamel
CaseLowerCamel
CaseUpperUnderscore
CaseLowerUnderscore
)
//ToCaseFormat format text, from, to are const: CaseLower, CaseUpperCamel, CaseLowerCamel, CaseUpperUnderscore, CaseLowerUnderscore,
func ToCaseFormat(text string, from, to int) string {
toUpper := false
toLower := false
toCamel := false
toUnserscore := false
fromCamel := false
fromUnserscore := false
switch to {
case CaseUpper, CaseUpperUnderscore:
toUpper = true
case CaseLower, CaseLowerUnderscore:
toLower = true
case CaseUpperCamel, CaseLowerCamel:
toCamel = true
}
switch to {
case CaseUpperUnderscore, CaseLowerUnderscore:
toUnserscore = true
}
switch from {
case CaseUpperCamel, CaseLowerCamel:
fromCamel = true
case CaseUpperUnderscore, CaseLowerUnderscore:
fromUnserscore = true
}
underscore := rune('_')
var result = make([]rune, 0)
makeLower := false
makeUpper := false
hasUnderscore := false
for i, r := range text {
first := i == 0
if toUpper {
makeUpper = true
} else if toLower {
makeLower = true
}
if first {
if to == CaseLowerCamel {
r = unicode.ToLower(r)
} else if to == CaseUpperCamel {
r = unicode.ToUpper(r)
}
} else {
if fromUnserscore {
if toCamel {
if r == underscore {
hasUnderscore = true
continue
}
if hasUnderscore {
makeUpper = true
hasUnderscore = false
} else {
makeLower = true
}
}
}
if unicode.IsUpper(r) && fromCamel {
if toUnserscore {
result = append(result, underscore)
}
}
}
if makeLower {
r = unicode.ToLower(r)
} else if makeUpper {
r = unicode.ToUpper(r)
}
result = append(result, r)
makeUpper = false
makeLower = false
}
return string(result)
}
toolbox-0.33.2/text_test.go 0000664 0000000 0000000 00000015264 13741102511 0015642 0 ustar 00root root 0000000 0000000 package toolbox
import (
"bytes"
"fmt"
"github.com/stretchr/testify/assert"
"io"
"strings"
"testing"
)
func TestIsASCIIText(t *testing.T) {
var useCases = []struct {
Description string
Candidate string
Expected bool
}{
{
Description: "basic text",
Candidate: `abc`,
Expected: true,
},
{
Description: "JSON object like text",
Candidate: `{"k1"}`,
Expected: true,
},
{
Description: "JSON array like text",
Candidate: `["$k1"]`,
Expected: true,
},
{
Description: "bin data",
Candidate: "\u0000",
Expected: false,
},
{
Description: "JSON text",
Candidate: `{
"RepositoryDatastore":"db1",
"Db": [
{
"Name": "db1",
"Config": {
"PoolSize": 3,
"MaxPoolSize": 5,
"DriverName": "mysql",
"Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/db1?parseTime=true",
"Credentials": "$mysqlCredentials"
}
}
]
}
`,
Expected: true,
},
}
for _, useCase := range useCases {
assert.EqualValues(t, useCase.Expected, IsASCIIText(useCase.Candidate), useCase.Description)
}
}
func TestIsPrintText(t *testing.T) {
var useCases = []struct {
Description string
Candidate string
Expected bool
}{
{
Description: "basic text",
Candidate: `abc`,
Expected: true,
},
{
Description: "JSON object like text",
Candidate: `{"k1"}`,
Expected: true,
},
{
Description: "JSON array like text",
Candidate: `["$k1"]`,
Expected: true,
},
{
Description: "bin data",
Candidate: "\u0000",
Expected: false,
},
{
Description: "JSON text",
Candidate: `{
"RepositoryDatastore":"db1",
"Db": [
{
"Name": "db1",
"Config": {
"PoolSize": 3,
"MaxPoolSize": 5,
"DriverName": "mysql",
"Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/db1?parseTime=true",
"Credentials": "mysql"
}
}
]
}
`,
Expected: true,
},
}
for _, useCase := range useCases {
assert.EqualValues(t, useCase.Expected, IsPrintText(useCase.Candidate), useCase.Description)
}
}
func TestTerminatedSplitN(t *testing.T) {
var data = make([]byte, 0)
for i := 0; i < 9; i++ {
data = append(data, []byte(fmt.Sprintf("%v %v\n", strings.Repeat("x", 32), i))...)
}
text := string(data)
useCases := []struct {
description string
fragmentCount int
expectedFragmentSizes []int
}{
{
description: "one fragment case",
fragmentCount: 1,
expectedFragmentSizes: []int{len(data)},
},
{
description: "two fragments case",
fragmentCount: 2,
expectedFragmentSizes: []int{175, 140},
},
{
description: "3 fragments case",
fragmentCount: 3,
expectedFragmentSizes: []int{140, 140, 35},
},
{
description: "7 fragments case",
fragmentCount: 7,
expectedFragmentSizes: []int{70, 70, 70, 70, 35},
},
{
description: "10 fragments case", //no more fragments then lines, so only 9 fragments here
fragmentCount: 10,
expectedFragmentSizes: []int{35, 35, 35, 35, 35, 35, 35, 35, 35},
},
}
for _, useCase := range useCases {
fragments := TerminatedSplitN(text, useCase.fragmentCount, "\n")
var actualFragmentSizes = make([]int, len(fragments))
for i, fragment := range fragments {
actualFragmentSizes[i] = len(fragment)
}
assert.EqualValues(t, useCase.expectedFragmentSizes, actualFragmentSizes, useCase.description)
}
}
type testWriter struct {
*bytes.Buffer
data *[]string
}
func (t *testWriter) Close() error {
*t.data = append(*t.data, t.String())
return nil
}
func newTestWriter(data *[]string) io.WriteCloser {
return &testWriter{
data: data,
Buffer: new(bytes.Buffer),
}
}
func TestSplitTextStream(t *testing.T) {
var data = make([]byte, 0)
for i := 0; i < 9; i++ {
data = append(data, []byte(fmt.Sprintf("%v %v\n", strings.Repeat("x", 2), i))...)
}
text := string(data)
useCases := []struct {
description string
elements int
expect []string
}{
{
description: "no more then 4 lines case",
elements: 4,
expect: []string{
"xx 0\nxx 1\nxx 2\nxx 3", "xx 4\nxx 5\nxx 6\nxx 7", "xx 8",
},
},
{
description: "3 elements each",
elements: 3,
expect: []string{
"xx 0\nxx 1\nxx 2", "xx 3\nxx 4\nxx 5", "xx 6\nxx 7\nxx 8",
},
},
{
description: "9 elements",
elements: 1,
expect: []string{
"xx 0", "xx 1", "xx 2", "xx 3", "xx 4", "xx 5", "xx 6", "xx 7", "xx 8",
},
},
{
description: "9 elements",
elements: 0,
expect: []string{
"xx 0", "xx 1", "xx 2", "xx 3", "xx 4", "xx 5", "xx 6", "xx 7", "xx 8",
},
},
{
description: "1 elements",
elements: 10,
expect: []string{
"xx 0\nxx 1\nxx 2\nxx 3\nxx 4\nxx 5\nxx 6\nxx 7\nxx 8",
},
},
{
description: "1 elements",
elements: 9,
expect: []string{
"xx 0\nxx 1\nxx 2\nxx 3\nxx 4\nxx 5\nxx 6\nxx 7\nxx 8",
},
},
}
for _, useCase := range useCases {
var data = make([]string, 0)
err := SplitTextStream(strings.NewReader(text), func() io.WriteCloser { return newTestWriter(&data) }, useCase.elements)
assert.Nil(t, err)
assert.EqualValues(t, useCase.expect, data, useCase.description)
}
}
func Test_CaseFormat(t *testing.T) {
var useCases = []struct {
description string
caseFrom int
caseTo int
input string
expect string
}{
{
description: "camel to uppercase",
input: "thisIsMyTest",
caseFrom: CaseLowerCamel,
caseTo: CaseUpper,
expect: "THISISMYTEST",
},
{
description: "camel to lower underscore",
input: "thisIsMyTest",
caseFrom: CaseLowerCamel,
caseTo: CaseLowerUnderscore,
expect: "this_is_my_test",
},
{
description: "camel to upper underscore",
input: "thisIsMyTest",
caseFrom: CaseLowerCamel,
caseTo: CaseUpperUnderscore,
expect: "THIS_IS_MY_TEST",
},
{
description: "lower underscore to upper camel",
input: "this_is_my_test",
caseFrom: CaseLowerUnderscore,
caseTo: CaseUpperCamel,
expect: "ThisIsMyTest",
},
{
description: "upper underscore to lower camel",
input: "THIS_IS_MY_TEST",
caseFrom: CaseUpperUnderscore,
caseTo: CaseLowerCamel,
expect: "thisIsMyTest",
},
{
description: "upper camel to lower camel",
input: "ThisIsMyTest",
caseFrom: CaseUpperCamel,
caseTo: CaseLowerCamel,
expect: "thisIsMyTest",
},
}
for _, useCase := range useCases {
actual := ToCaseFormat(useCase.input, useCase.caseFrom, useCase.caseTo)
assert.Equal(t, useCase.expect, actual, useCase.description)
}
}
toolbox-0.33.2/time_format.go 0000664 0000000 0000000 00000007106 13741102511 0016121 0 ustar 00root root 0000000 0000000 package toolbox
import (
"strings"
"time"
)
//DateFormatKeyword constant 'dateFormat' key
var DateFormatKeyword = "dateFormat"
//DateLayoutKeyword constant 'dateLayout' key
var DateLayoutKeyword = "dateLayout"
//DateFormatToLayout converts java date format https://docs.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html#rfc822timezone into go date layout
func DateFormatToLayout(dateFormat string) string {
dateFormat = strings.Replace(dateFormat, "ddd", "_2", 1)
dateFormat = strings.Replace(dateFormat, "dd", "02", 1)
dateFormat = strings.Replace(dateFormat, "d", "2", 1)
dateFormat = strings.Replace(dateFormat, "HH", "15", 1)
dateFormat = strings.Replace(dateFormat, "hh", "03", 1)
dateFormat = strings.Replace(dateFormat, "h", "3", 1)
dateFormat = strings.Replace(dateFormat, "mm", "04", 1)
dateFormat = strings.Replace(dateFormat, "m", "4", 1)
dateFormat = strings.Replace(dateFormat, "ss", "05", 1)
dateFormat = strings.Replace(dateFormat, "s", "5", 1)
dateFormat = strings.Replace(dateFormat, "yyyy", "2006", 1)
dateFormat = strings.Replace(dateFormat, "yy", "06", 1)
dateFormat = strings.Replace(dateFormat, "y", "06", 1)
dateFormat = strings.Replace(dateFormat, "SSS", "000", 1)
dateFormat = strings.Replace(dateFormat, "a", "pm", 1)
dateFormat = strings.Replace(dateFormat, "aa", "PM", 1)
dateFormat = strings.Replace(dateFormat, "MMMM", "January", 1)
dateFormat = strings.Replace(dateFormat, "MMM", "Jan", 1)
dateFormat = strings.Replace(dateFormat, "MM", "01", 1)
dateFormat = strings.Replace(dateFormat, "M", "1", 1)
dateFormat = strings.Replace(dateFormat, "ZZ", "-0700", 1)
dateFormat = strings.Replace(dateFormat, "Z", "-07", 1)
dateFormat = strings.Replace(dateFormat, "zz:zz", "Z07:00", 1)
dateFormat = strings.Replace(dateFormat, "zzzz", "Z0700", 1)
dateFormat = strings.Replace(dateFormat, "z", "MST", 1)
dateFormat = strings.Replace(dateFormat, "EEEE", "Monday", 1)
dateFormat = strings.Replace(dateFormat, "E", "Mon", 1)
return dateFormat
}
//GetTimeLayout returns time laout from passed in map, first it check if DateLayoutKeyword is defined is so it returns it, otherwise it check DateFormatKeyword and if exists converts it to dateLayout
//If neithers keys exists it panics, please use HasTimeLayout to avoid panic
func GetTimeLayout(input interface{}) string {
switch settings := input.(type) {
case map[string]string:
if value, found := settings[DateLayoutKeyword]; found {
return value
}
if value, found := settings[DateFormatKeyword]; found {
return DateFormatToLayout(value)
}
case map[string]interface{}:
if value, found := settings[DateLayoutKeyword]; found {
return AsString(value)
}
if value, found := settings[DateFormatKeyword]; found {
return DateFormatToLayout(AsString(value))
}
}
return ""
}
//HasTimeLayout checks if dateLayout can be taken from the passed in setting map
func HasTimeLayout(input interface{}) bool {
switch settings := input.(type) {
case map[string]string:
if _, found := settings[DateLayoutKeyword]; found {
return true
}
if _, found := settings[DateFormatKeyword]; found {
return true
}
case map[string]interface{}:
if _, found := settings[DateLayoutKeyword]; found {
return true
}
if _, found := settings[DateFormatKeyword]; found {
return true
}
}
return false
}
//TimestampToString formats timestamp to passed in java style date format
func TimestampToString(dateFormat string, unixTimestamp, unixNanoTimestamp int64) string {
t := time.Unix(unixTimestamp, unixNanoTimestamp)
dateLayout := DateFormatToLayout(dateFormat)
return t.Format(dateLayout)
}
toolbox-0.33.2/time_format_test.go 0000664 0000000 0000000 00000007222 13741102511 0017157 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"fmt"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"strings"
"testing"
"time"
)
func TestTimeFormat(t *testing.T) {
{
timeLayout := toolbox.DateFormatToLayout("yyyy/MM/dd/hh")
fmt.Printf("%s\n", timeLayout)
}
{
timeLayout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss z")
timeValue, err := time.Parse(timeLayout, "2018-01-15 08:02:23 UTC")
assert.Nil(t, err)
assert.EqualValues(t, 23, timeValue.Second())
}
{
timeLayout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss")
timeValue, err := time.Parse(timeLayout, "2016-03-01 03:10:11")
assert.Nil(t, err)
assert.EqualValues(t, 11, timeValue.Second())
}
{
dateLayout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss.SSSZ")
timeValue, err := time.Parse(dateLayout, "2022-11-10 10:32:28.984-08")
assert.Nil(t, err)
assert.Equal(t, int64(1668105148), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss.SSSZ")
timeValue, err := time.Parse(dateLaout, "2022-11-10 10:32:28.984-08")
assert.Nil(t, err)
assert.Equal(t, int64(1668105148), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss.SSS")
timeValue, err := time.Parse(dateLaout, "2017-11-04 22:29:33.363")
assert.Nil(t, err)
assert.Equal(t, 2017, timeValue.Year())
assert.Equal(t, time.Month(11), timeValue.Month())
assert.Equal(t, 4, timeValue.Day())
assert.Equal(t, int64(1509834573), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("dd/MM/yyyy hh:mm:ss")
timeValue, err := time.Parse(dateLaout, "22/02/2016 12:32:01")
assert.Nil(t, err)
assert.Equal(t, int64(1456144321), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyyMMdd hh:mm:ss")
timeValue, err := time.Parse(dateLaout, "20160222 12:32:01")
assert.Nil(t, err)
assert.Equal(t, int64(1456144321), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss z")
timeValue, err := time.Parse(dateLaout, "2016-02-22 12:32:01 UTC")
assert.Nil(t, err)
assert.Equal(t, int64(1456144321), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss z")
timeValue, err := time.Parse(dateLaout, "2016-02-22 12:32:01 UTC")
assert.Nil(t, err)
assert.Equal(t, int64(1456144321), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss z")
timeValue, err := time.Parse(dateLaout, "2016-02-22 12:32:01 UTC")
assert.Nil(t, err)
assert.Equal(t, int64(1456144321), timeValue.Unix())
}
{
dateLaout := toolbox.DateFormatToLayout("yyyy-MM-dd HH:mm:ss z")
timeValue, err := time.Parse(dateLaout, "2016-06-02 21:46:19 UTC")
assert.Nil(t, err)
assert.Equal(t, int64(1464903979), timeValue.Unix())
}
}
func TestGetTimeLayout(t *testing.T) {
{
settings := map[string]string{
toolbox.DateFormatKeyword: "yyyy-MM-dd HH:mm:ss z",
}
assert.Equal(t, "2006-01-02 15:04:05 MST", toolbox.GetTimeLayout(settings))
assert.True(t, toolbox.HasTimeLayout(settings))
}
{
settings := map[string]string{
toolbox.DateLayoutKeyword: "2006-1-02 15:04:05 MST",
}
assert.Equal(t, "2006-1-02 15:04:05 MST", toolbox.GetTimeLayout(settings))
assert.True(t, toolbox.HasTimeLayout(settings))
}
{
settings := map[string]string{}
assert.False(t, toolbox.HasTimeLayout(settings))
}
}
func TestTimestampToString(t *testing.T) {
{
date := toolbox.TimestampToString("yyyy-MM-dd HH:mm:ss z", int64(0), 1480435743722684356)
assert.True(t, strings.Contains(date, "2016-11"))
}
{
date := toolbox.TimestampToString("yyyyMMddhh", int64(0), 1489512277722684356)
assert.True(t, strings.Contains(date, "201703"))
}
}
toolbox-0.33.2/time_helper.go 0000664 0000000 0000000 00000026262 13741102511 0016114 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"log"
"strings"
"time"
)
const (
Now = "now"
Tomorrow = "tomorrow"
Yesterday = "yesterday"
//TimeAtTwoHoursAgo = "2hoursAgo"
//TimeAtHourAhead = "hourAhead"
//TimeAtTwoHoursAhead = "2hoursAhead"
DurationWeek = "week"
DurationDay = "day"
DurationHour = "hour"
DurationMinute = "minute"
DurationMinuteAbbr = "min"
DurationSecond = "second"
DurationSecondAbbr = "sec"
DurationMillisecond = "millisecond"
DurationMillisecondAbbr = "ms"
DurationMicrosecond = "microsecond"
DurationMicrosecondAbbr = "us"
DurationNanosecond = "nanosecond"
DurationNanosecondAbbr = "ns"
)
//AtTime represents a time at given schedule
type AtTime struct {
WeekDay string
Hour string
Minute string
TZ string
loc *time.Location
}
func (t *AtTime) min(base time.Time) int {
switch t.Minute {
case "*":
return (base.Minute() + 1) % 59
case "":
return 0
}
candidates := strings.Split(t.Minute, ",")
for _, candidate := range candidates {
candidateMin := AsInt(candidate)
if base.Minute() < candidateMin {
return candidateMin
}
}
return AsInt(candidates[0])
}
func (t *AtTime) hour(base time.Time) int {
min := t.min(base)
switch t.Hour {
case "*":
if min > base.Minute() {
return base.Hour()
}
return (base.Hour() + 1) % 23
case "":
return 0
}
candidates := strings.Split(t.Hour, ",")
for _, candidate := range candidates {
candidateHour := AsInt(candidate)
if base.Hour() < candidateHour {
return candidateHour
}
}
return AsInt(candidates[0])
}
func (t *AtTime) weekday(base time.Time) int {
hour := t.hour(base)
min := t.min(base)
baseWeekday := int(base.Weekday())
isPastDue := hour > base.Hour() || (hour == base.Hour() && min > base.Minute())
switch t.WeekDay {
case "*":
if isPastDue {
return baseWeekday
}
return (baseWeekday + 1) % 7
case "":
return 0
}
candidates := strings.Split(t.WeekDay, ",")
result := AsInt(candidates[0])
for _, candidate := range candidates {
candidateWeekday := AsInt(candidate)
if baseWeekday < candidateWeekday {
result = candidateWeekday
break
}
}
if result < baseWeekday && isPastDue {
return 7 + result
}
return result
}
//Init initializes tz
func (t *AtTime) Init() error {
if t.TZ == "" {
return nil
}
var err error
t.loc, err = time.LoadLocation(t.TZ)
return err
}
//Next returns next time schedule
func (t *AtTime) Next(base time.Time) time.Time {
if t.loc != nil && base.Location() != nil && base.Location() != t.loc {
base = base.In(t.loc)
} else {
t.loc = base.Location()
}
min := t.min(base)
hour := t.hour(base)
timeLiteral := base.Format("2006-01-02")
updateTimeLiteral := fmt.Sprintf("%v %02d:%02d:00", timeLiteral, hour, min)
weekday := t.weekday(base)
baseWeekday := int(base.Weekday())
weekdayDiff := 0
if weekday >= baseWeekday {
weekdayDiff = weekday - baseWeekday
} else {
weekdayDiff = 7 + weekday - baseWeekday
}
var result time.Time
if t.loc != nil {
result, _ = time.ParseInLocation("2006-01-02 15:04:05", updateTimeLiteral, t.loc)
} else {
result, _ = time.Parse("2006-01-02 15:04:05", updateTimeLiteral)
}
if weekdayDiff > 0 {
result = result.Add(time.Hour * 24 * time.Duration(weekdayDiff))
} else if weekdayDiff == 0 && AsInt(t.WeekDay) > 0 {
result = result.Add(time.Hour * 24 * 7)
}
if result.UnixNano() < base.UnixNano() {
log.Printf("invalid schedule next: %v is before base: %v\n", result, base)
}
return result
}
//Duration represents duration
type Duration struct {
Value int
Unit string
}
//Duration return durations
func (d Duration) Duration() (time.Duration, error) {
return NewDuration(d.Value, d.Unit)
}
//NewDuration returns a durationToken for supplied value and time unit, 3, "second"
func NewDuration(value int, unit string) (time.Duration, error) {
var duration time.Duration
switch unit {
case DurationWeek:
duration = time.Hour * 24 * 7
case DurationDay:
duration = time.Hour * 24
case DurationHour:
duration = time.Hour
case DurationMinute, DurationMinuteAbbr:
duration = time.Minute
case DurationSecond, DurationSecondAbbr:
duration = time.Second
case DurationMillisecond, DurationMillisecondAbbr:
duration = time.Millisecond
case DurationMicrosecond, DurationMicrosecondAbbr:
duration = time.Microsecond
case DurationNanosecond, DurationNanosecondAbbr:
duration = time.Nanosecond
default:
return 0, fmt.Errorf("unsupported unit: %v", unit)
}
return time.Duration(value) * duration, nil
}
const (
eofToken = -1
invalidToken = iota
timeValueToken
nowToken
yesterdayToken
tomorrowToken
whitespacesToken
durationToken
inTimezoneToken
durationPluralToken
positiveModifierToken
negativeModifierToken
timezoneToken
)
var timeAtExpressionMatchers = map[int]Matcher{
timeValueToken: NewIntMatcher(),
whitespacesToken: CharactersMatcher{" \n\t"},
durationToken: NewKeywordsMatcher(false, DurationWeek, DurationDay, DurationHour, DurationMinute, DurationMinuteAbbr, DurationSecond, DurationSecondAbbr, DurationMillisecond, DurationMillisecondAbbr, DurationMicrosecond, DurationMicrosecondAbbr, DurationNanosecond, DurationNanosecondAbbr),
durationPluralToken: NewKeywordsMatcher(false, "s"),
nowToken: NewKeywordsMatcher(false, Now),
yesterdayToken: NewKeywordsMatcher(false, Yesterday),
tomorrowToken: NewKeywordsMatcher(false, Tomorrow),
positiveModifierToken: NewKeywordsMatcher(false, "onward", "ahead", "after", "later", "in the future", "inthefuture"),
negativeModifierToken: NewKeywordsMatcher(false, "past", "ago", "before", "earlier", "in the past", "inthepast"),
inTimezoneToken: NewKeywordsMatcher(false, "in"),
timezoneToken: NewRemainingSequenceMatcher(),
eofToken: &EOFMatcher{},
}
//TimeAt returns time of now supplied offsetExpression, this function uses TimeDiff
func TimeAt(offsetExpression string) (*time.Time, error) {
return TimeDiff(time.Now(), offsetExpression)
}
//TimeDiff returns time for supplied base time and literal, the supported literal now, yesterday, tomorrow, or the following template:
// - [timeValueToken] durationToken past_or_future_modifier [IN tz]
// where time modifier can be any of the following: "onward", "ahead", "after", "later", or "past", "ago", "before", "earlier", "in the future", "in the past") )
func TimeDiff(base time.Time, expression string) (*time.Time, error) {
if expression == "" {
return nil, fmt.Errorf("expression was empty")
}
var delta time.Duration
var isNegative = false
tokenizer := NewTokenizer(expression, invalidToken, eofToken, timeAtExpressionMatchers)
var val = 1
var isTimeExtracted = false
token, err := ExpectToken(tokenizer, "", timeValueToken, nowToken, yesterdayToken, tomorrowToken)
if err == nil {
switch token.Token {
case timeValueToken:
val, _ = ToInt(token.Matched)
case yesterdayToken:
isNegative = true
fallthrough
case tomorrowToken:
delta, _ = NewDuration(1, DurationDay)
fallthrough
case nowToken:
isTimeExtracted = true
}
}
if !isTimeExtracted {
token, err = ExpectTokenOptionallyFollowedBy(tokenizer, whitespacesToken, "expected time unit", durationToken)
if err != nil {
return nil, err
}
delta, _ = NewDuration(val, strings.ToLower(token.Matched))
_, _ = ExpectToken(tokenizer, "", durationPluralToken)
token, err = ExpectTokenOptionallyFollowedBy(tokenizer, whitespacesToken, "expected time modifier", positiveModifierToken, negativeModifierToken)
if err != nil {
return nil, err
}
if token.Token == negativeModifierToken {
isNegative = true
}
}
if token, err = ExpectTokenOptionallyFollowedBy(tokenizer, whitespacesToken, "expected in", inTimezoneToken); err == nil {
token, err = ExpectToken(tokenizer, "epected timezone", timezoneToken)
if err != nil {
return nil, err
}
tz := strings.TrimSpace(token.Matched)
tzLocation, err := time.LoadLocation(tz)
if err != nil {
return nil, fmt.Errorf("failed to load timezone tzLocation: %v, %v", tz, err)
}
base = base.In(tzLocation)
}
token, err = ExpectToken(tokenizer, "expected eofToken", eofToken)
if err != nil {
return nil, err
}
if isNegative {
delta *= -1
}
base = base.Add(delta)
return &base, nil
}
//ElapsedToday returns elapsed today time percent, it takes optionally timezone
func ElapsedToday(tz string) (float64, error) {
if tz != "" {
tz = "In" + tz
}
now, err := TimeAt("now" + tz)
if err != nil {
return 0, err
}
return ElapsedDay(*now), nil
}
//ElapsedDay returns elapsed pct for passed in day (second elapsed that day over 24 hours)
func ElapsedDay(dateTime time.Time) float64 {
elapsedToday := time.Duration(dateTime.Hour())*time.Hour + time.Duration(dateTime.Minute())*time.Minute + time.Duration(dateTime.Second()) + time.Second
elapsedTodayPct := float64(elapsedToday) / float64((24 * time.Hour))
return elapsedTodayPct
}
//RemainingToday returns remaining today time percent, it takes optionally timezone
func RemainingToday(tz string) (float64, error) {
elapsedToday, err := ElapsedToday(tz)
if err != nil {
return 0, err
}
return 1.0 - elapsedToday, nil
}
//TimeWindow represents a time window
type TimeWindow struct {
Loopback *Duration
StartDate string
startTime *time.Time
EndDate string
endTime *time.Time
TimeLayout string
TimeFormat string
Interval *Duration
}
//Range iterates with interval step between start and window end.
func (w *TimeWindow) Range(handler func(time time.Time) (bool, error)) error {
start, err := w.StartTime()
if err != nil {
return err
}
end, err := w.EndTime()
if err != nil {
return err
}
if w.Interval == nil && w.Loopback != nil {
w.Interval = w.Loopback
}
if w.Interval == nil {
_, err = handler(*end)
return err
}
interval, err := w.Interval.Duration()
if err != nil {
return err
}
for ts := *start; ts.Before(*end) || ts.Equal(*end); ts = ts.Add(interval) {
if ok, err := handler(ts); err != nil || !ok {
return err
}
}
return err
}
//Layout return time layout
func (w *TimeWindow) Layout() string {
if w.TimeLayout != "" {
return w.TimeLayout
}
if w.TimeFormat != "" {
w.TimeLayout = DateFormatToLayout(w.TimeFormat)
}
if w.TimeLayout == "" {
w.TimeLayout = time.RFC3339
}
return w.TimeLayout
}
//StartTime returns time window start time
func (w *TimeWindow) StartTime() (*time.Time, error) {
if w.StartDate != "" {
if w.startTime != nil {
return w.startTime, nil
}
timeLayout := w.Layout()
startTime, err := time.Parse(timeLayout, w.StartDate)
if err != nil {
return nil, err
}
w.startTime = &startTime
return w.startTime, nil
}
endDate, err := w.EndTime()
if err != nil {
return nil, err
}
if w.Loopback == nil || w.Loopback.Value == 0 {
return endDate, nil
}
loopback, err := w.Loopback.Duration()
if err != nil {
return nil, err
}
startTime := endDate.Add(-loopback)
return &startTime, nil
}
//EndTime returns time window end time
func (w *TimeWindow) EndTime() (*time.Time, error) {
if w.EndDate != "" {
if w.endTime != nil {
return w.endTime, nil
}
timeLayout := w.Layout()
endTime, err := time.Parse(timeLayout, w.EndDate)
if err != nil {
return nil, err
}
w.endTime = &endTime
return w.endTime, nil
}
now := time.Now()
return &now, nil
}
toolbox-0.33.2/time_helper_test.go 0000664 0000000 0000000 00000026141 13741102511 0017147 0 ustar 00root root 0000000 0000000 package toolbox
import (
"github.com/stretchr/testify/assert"
"math"
"testing"
"time"
)
func TestAtTime_Next(t *testing.T) {
timeLayout := "2006-01-02 15:04:05"
var useCases = []struct {
description string
at *AtTime
baseTime string
expectTime string
}{
{
description: "evey 1/2 - next day",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "30",
},
baseTime: "2019-01-01 23:33:01",
expectTime: "2019-01-02 01:30:00",
},
{
description: "evey 1/2, last minute - next day",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "0,30",
},
baseTime: "2019-01-01 23:59:00",
expectTime: "2019-01-02 01:00:00",
},
{
description: "evey 1/2, last day of the month - next day",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "0,30",
},
baseTime: "2019-05-31 23:59:00",
expectTime: "2019-06-01 01:00:00",
},
{
description: "evey hour - next day",
at: &AtTime{
WeekDay: "*",
Hour: "0",
Minute: "*",
},
baseTime: "2019-01-01 23:01:01",
expectTime: "2019-01-02 00:02:00",
},
{
description: "evey minute",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "*",
},
baseTime: "2019-01-01 01:01:01",
expectTime: "2019-01-01 01:02:00",
},
{
description: "evey 30 minute before",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "30",
},
baseTime: "2019-01-01 01:01:01",
expectTime: "2019-01-01 01:30:00",
},
{
description: "evey 30 minute after",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "30",
},
baseTime: "2019-01-01 01:31:01",
expectTime: "2019-01-01 02:30:00",
},
{
description: "evey 10, 30 minute, before",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "10,30",
},
baseTime: "2019-01-01 01:09:01",
expectTime: "2019-01-01 01:10:00",
},
{
description: "evey 10, 30 minute, after first",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "10,30",
},
baseTime: "2019-01-01 01:13:01",
expectTime: "2019-01-01 01:30:00",
},
{
description: "evey 10, 30 minute, after second",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "10,30",
},
baseTime: "2019-01-01 01:33:01",
expectTime: "2019-01-01 02:10:00",
},
{
description: "evey *:0 minute",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "0",
},
baseTime: "2019-01-01 01:59:01",
expectTime: "2019-01-01 02:00:00",
},
{
description: "evey hour",
at: &AtTime{
WeekDay: "*",
Hour: "*",
Minute: "",
},
baseTime: "2019-01-01 01:33:01",
expectTime: "2019-01-01 02:00:00",
},
{
description: "at 13 hour, before",
at: &AtTime{
WeekDay: "*",
Hour: "13",
Minute: "",
},
baseTime: "2019-01-01 01:33:01",
expectTime: "2019-01-01 13:00:00",
},
{
description: "at 13 hour, after",
at: &AtTime{
WeekDay: "*",
Hour: "13",
Minute: "",
},
baseTime: "2019-01-01 15:33:01",
expectTime: "2019-01-02 13:00:00",
},
{
description: "at midnight",
at: &AtTime{
WeekDay: "*",
Hour: "0",
Minute: "",
},
baseTime: "2019-01-01 23:33:01",
expectTime: "2019-01-02 00:00:00",
},
{
description: "at midnight weekday 9",
at: &AtTime{
WeekDay: "*",
Hour: "0",
Minute: "",
},
baseTime: "2019-01-06 23:33:01",
expectTime: "2019-01-07 00:00:00",
},
{
description: "every 0 weekday",
at: &AtTime{
WeekDay: "0",
Hour: "",
Minute: "",
},
baseTime: "2019-01-04 23:33:01",
expectTime: "2019-01-06 00:00:00",
},
{
description: "every 2 weekday",
at: &AtTime{
WeekDay: "2",
Hour: "",
Minute: "",
},
baseTime: "2019-01-04 23:33:01",
expectTime: "2019-01-08 00:00:00",
},
{
description: "every 2nd weekday - overlaps with base time",
at: &AtTime{
WeekDay: "2",
Hour: "",
Minute: "",
},
baseTime: "2019-01-08 23:33:01",
expectTime: "2019-01-15 00:00:00",
},
{
description: "every 5 weekday in the future",
at: &AtTime{
WeekDay: "2,5",
Hour: "",
Minute: "",
},
baseTime: "2019-01-09 23:33:01",
expectTime: "2019-01-11 00:00:00",
},
{
description: "every 5 weekday in the future tz",
at: &AtTime{
WeekDay: "2,5",
Hour: "",
Minute: "",
TZ: "America/Los_Angeles",
},
baseTime: "2019-01-09 23:33:01",
expectTime: "2019-01-11 00:00:00",
},
}
for _, useCase := range useCases {
err := useCase.at.Init()
assert.Nil(t, err)
var loc *time.Location
if useCase.at.TZ != "" {
loc, _ = time.LoadLocation(useCase.at.TZ)
}
var baseTime time.Time
if loc != nil {
baseTime, err = time.ParseInLocation(timeLayout, useCase.baseTime, loc)
assert.Nil(t, err, useCase.description)
} else {
baseTime, err = time.Parse(timeLayout, useCase.baseTime)
assert.Nil(t, err, useCase.description)
}
var expectTime time.Time
if loc != nil {
expectTime, err = time.ParseInLocation(timeLayout, useCase.expectTime, loc)
assert.Nil(t, err, useCase.description)
} else {
expectTime, err = time.Parse(timeLayout, useCase.expectTime)
assert.Nil(t, err, useCase.description)
}
actualTime := useCase.at.Next(baseTime)
assert.Equal(t, expectTime, actualTime, useCase.description)
//without tz
baseTime, err = time.Parse(timeLayout, useCase.baseTime)
actualTime = useCase.at.Next(baseTime)
assert.Equal(t, expectTime, actualTime, useCase.description)
}
}
func TestNewDuration(t *testing.T) {
var useCases = []struct {
description string
value int
unit string
expected time.Duration
hasError bool
}{
{
description: "sec test",
value: 3,
unit: DurationSecond,
expected: 3 * time.Second,
},
{
description: "min test",
value: 4,
unit: DurationMinute,
expected: 4 * time.Minute,
},
{
description: "hour test",
value: 5,
unit: DurationHour,
expected: 5 * time.Hour,
},
{
description: "day test",
value: 12,
unit: DurationDay,
expected: 12 * time.Hour * 24,
},
{
description: "week test",
value: 7,
unit: DurationWeek,
expected: 7 * time.Hour * 24 * 7,
},
{
description: "error test",
value: 4,
unit: "abc",
hasError: true,
},
}
for _, useCase := range useCases {
actual, err := NewDuration(useCase.value, useCase.unit)
if useCase.hasError {
assert.NotNil(t, err, useCase.description)
continue
} else if err != nil {
assert.Nil(t, err, useCase.description)
continue
}
assert.Equal(t, useCase.expected, actual, useCase.description)
}
}
func TestIdMatcher_Match(t *testing.T) {
{
ts, err := TimeAt("1 sec ahead")
assert.Nil(t, err)
assert.EqualValues(t, ts.Unix()-1, time.Now().Unix())
}
{ //invalid duration unit
_, err := TimeAt("1 d ahead")
assert.NotNil(t, err)
}
}
func TestTimeDiff(t *testing.T) {
var useCases = []struct {
description string
base time.Time
expression string
exectedDiff time.Duration
hasError bool
}{
{
description: "now test",
expression: "now",
base: time.Now(),
exectedDiff: 0,
},
{
description: "tomorrow test",
expression: "tomorrow",
base: time.Now(),
exectedDiff: time.Hour * 24,
},
{
description: "yesterday test",
expression: "yesterday",
base: time.Now(),
exectedDiff: -time.Hour * 24,
},
{
description: "empty expr error",
hasError: true,
},
{
description: "parsing expr error",
expression: "a232",
hasError: true,
},
{
description: "2 days ago test",
expression: "2daysago",
base: time.Now(),
exectedDiff: -time.Hour * 48,
},
{
description: "2 days in the future",
expression: "2day in the future",
base: time.Now(),
exectedDiff: time.Hour * 48,
},
{
description: "days in the future",
expression: "day InTheFuture",
base: time.Now(),
exectedDiff: time.Hour * 24,
},
{
description: "2 hours before",
expression: "2hourbefore",
base: time.Now(),
exectedDiff: -time.Hour * 2,
},
{
description: "2 hours later",
expression: "2 hoursLater",
base: time.Now(),
exectedDiff: time.Hour * 2,
},
{
description: "timezone",
expression: "nowInUTC",
base: time.Now(),
exectedDiff: 0,
},
{
description: "invalid timezone error",
expression: "nowInBAAA",
base: time.Now(),
hasError: true,
},
{
description: "day in UTC",
expression: "2 days ago in UTC",
base: time.Now(),
exectedDiff: -time.Hour * 48,
},
{
description: "day in UTC",
expression: "weekAheadInUTC",
base: time.Now(),
exectedDiff: time.Hour * 24 * 7,
},
}
for _, useCase := range useCases {
actual, err := TimeDiff(useCase.base, useCase.expression)
if useCase.hasError {
assert.NotNil(t, err, useCase.description)
continue
} else if err != nil {
assert.Nil(t, err, useCase.description)
continue
}
expected := useCase.base.Add(useCase.exectedDiff)
assert.EqualValues(t, expected.Unix(), actual.Unix(), useCase.description)
}
}
func TestDayElapsedInPct(t *testing.T) {
t0, _ := time.Parse(DateFormatToLayout("yyyy-MM-dd hh:mm:ss"), "2017-01-01 12:00:00")
elapsedPct := ElapsedDay(t0)
assert.EqualValues(t, 50, math.Round(100*elapsedPct))
elapsed, err := ElapsedToday("")
assert.Nil(t, err)
assert.True(t, elapsed > 0)
remaining, err := RemainingToday("")
assert.Nil(t, err)
assert.True(t, remaining > 0)
assert.EqualValues(t, int(remaining+elapsed), 1)
}
func TestTimeWindow_Range(t *testing.T) {
var useCaes = []struct {
description string
window *TimeWindow
expectedCount int
}{
{
description: "empty window",
window: &TimeWindow{},
expectedCount: 1,
},
{
description: "loopback window",
window: &TimeWindow{
TimeFormat: "yyyy-MM-dd HH:mm:ss",
Loopback: &Duration{Value: 3, Unit: "sec"},
EndDate: "2011-12-01 15:01:01",
Interval: &Duration{Value: 1, Unit: "sec"},
},
expectedCount: 4,
},
{
description: "default loopback with interval window",
window: &TimeWindow{
Loopback: &Duration{Value: 3, Unit: "min"},
Interval: &Duration{Value: 1, Unit: "min"},
},
expectedCount: 4,
},
{
description: "default loopback window",
window: &TimeWindow{
Loopback: &Duration{Value: 3, Unit: "min"},
},
expectedCount: 2,
},
{
description: "date range window",
window: &TimeWindow{
TimeFormat: "yyyy-MM-dd HH:mm:ss",
StartDate: "2011-12-01 15:01:01",
EndDate: "2011-12-01 15:02:01",
Interval: &Duration{Value: 10, Unit: "sec"}},
expectedCount: 7,
},
}
for _, useCase := range useCaes {
count := 0
err := useCase.window.Range(func(time time.Time) (bool, error) {
count++
return true, nil
})
assert.Nil(t, err, useCase.description)
assert.Equal(t, useCase.expectedCount, count, useCase.description)
}
}
func TestN(t *testing.T) {
} toolbox-0.33.2/tokenizer.go 0000664 0000000 0000000 00000035360 13741102511 0015630 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"strings"
"unicode"
)
//Matcher represents a matcher, that matches input from offset position, it returns number of characters matched.
type Matcher interface {
//Match matches input starting from offset, it return number of characters matched
Match(input string, offset int) (matched int)
}
//Token a matchable input
type Token struct {
Token int
Matched string
}
//Tokenizer represents a token scanner.
type Tokenizer struct {
matchers map[int]Matcher
Input string
Index int
InvalidToken int
EndOfFileToken int
}
//Nexts matches the first of the candidates
func (t *Tokenizer) Nexts(candidates ...int) *Token {
for _, candidate := range candidates {
result := t.Next(candidate)
if result.Token != t.InvalidToken {
return result
}
}
return &Token{t.InvalidToken, ""}
}
//Next tries to match a candidate, it returns token if imatching is successful.
func (t *Tokenizer) Next(candidate int) *Token {
offset := t.Index
if !(offset < len(t.Input)) {
return &Token{t.EndOfFileToken, ""}
}
if candidate == t.EndOfFileToken {
return &Token{t.InvalidToken, ""}
}
if matcher, ok := t.matchers[candidate]; ok {
matchedSize := matcher.Match(t.Input, offset)
if matchedSize > 0 {
t.Index = t.Index + matchedSize
return &Token{candidate, t.Input[offset : offset+matchedSize]}
}
} else {
panic(fmt.Sprintf("failed to lookup matcher for %v", candidate))
}
return &Token{t.InvalidToken, ""}
}
//NewTokenizer creates a new NewTokenizer, it takes input, invalidToken, endOfFileToeken, and matchers.
func NewTokenizer(input string, invalidToken int, endOfFileToken int, matcher map[int]Matcher) *Tokenizer {
return &Tokenizer{
matchers: matcher,
Input: input,
Index: 0,
InvalidToken: invalidToken,
EndOfFileToken: endOfFileToken,
}
}
//CharactersMatcher represents a matcher, that matches any of Chars.
type CharactersMatcher struct {
Chars string //characters to be matched
}
//Match matches any characters defined in Chars in the input, returns 1 if character has been matched
func (m CharactersMatcher) Match(input string, offset int) int {
var matched = 0
if offset >= len(input) {
return matched
}
outer:
for _, r := range input[offset:] {
for _, candidate := range m.Chars {
if candidate == r {
matched++
continue outer
}
}
break
}
return matched
}
//NewCharactersMatcher creates a new character matcher
func NewCharactersMatcher(chars string) Matcher {
return &CharactersMatcher{Chars: chars}
}
//EOFMatcher represents end of input matcher
type EOFMatcher struct {
}
//Match returns 1 if end of input has been reached otherwise 0
func (m EOFMatcher) Match(input string, offset int) int {
if offset+1 == len(input) {
return 1
}
return 0
}
//IntMatcher represents a matcher that finds any int in the input
type IntMatcher struct{}
//Match matches a literal in the input, it returns number of character matched.
func (m IntMatcher) Match(input string, offset int) int {
var matched = 0
if offset >= len(input) {
return matched
}
for _, r := range input[offset:] {
if !unicode.IsDigit(r) {
break
}
matched++
}
return matched
}
//NewIntMatcher returns a new integer matcher
func NewIntMatcher() Matcher {
return &IntMatcher{}
}
var dotRune = rune('.')
var underscoreRune = rune('_')
//LiteralMatcher represents a matcher that finds any literals in the input
type LiteralMatcher struct{}
//Match matches a literal in the input, it returns number of character matched.
func (m LiteralMatcher) Match(input string, offset int) int {
var matched = 0
if offset >= len(input) {
return matched
}
for i, r := range input[offset:] {
if i == 0 {
if !unicode.IsLetter(r) {
break
}
} else if !(unicode.IsLetter(r) || unicode.IsDigit(r) || r == dotRune || r == underscoreRune) {
break
}
matched++
}
return matched
}
//LiteralMatcher represents a matcher that finds any literals in the input
type IdMatcher struct{}
//Match matches a literal in the input, it returns number of character matched.
func (m IdMatcher) Match(input string, offset int) int {
var matched = 0
if offset >= len(input) {
return matched
}
for i, r := range input[offset:] {
if i == 0 {
if !(unicode.IsLetter(r) || unicode.IsDigit(r)) {
break
}
} else if !(unicode.IsLetter(r) || unicode.IsDigit(r) || r == dotRune || r == underscoreRune) {
break
}
matched++
}
return matched
}
//SequenceMatcher represents a matcher that finds any sequence until find provided terminators
type SequenceMatcher struct {
Terminators []string
CaseSensitive bool
matchAllIfNoTerminator bool
runeTerminators []rune
}
func (m *SequenceMatcher) hasTerminator(candidate string) bool {
var candidateLength = len(candidate)
for _, terminator := range m.Terminators {
terminatorLength := len(terminator)
if len(terminator) > candidateLength {
continue
}
if !m.CaseSensitive {
if strings.ToLower(terminator) == strings.ToLower(string(candidate[:terminatorLength])) {
return true
}
}
if terminator == string(candidate[:terminatorLength]) {
return true
}
}
return false
}
//Match matches a literal in the input, it returns number of character matched.
func (m *SequenceMatcher) Match(input string, offset int) int {
var matched = 0
hasTerminator := false
if offset >= len(input) {
return matched
}
if len(m.runeTerminators) > 0 {
return m.matchSingleTerminator(input, offset)
}
var i = 0
for ; i < len(input)-offset; i++ {
if m.hasTerminator(string(input[offset+i:])) {
hasTerminator = true
break
}
}
if !hasTerminator && !m.matchAllIfNoTerminator {
return 0
}
return i
}
func (m *SequenceMatcher) matchSingleTerminator(input string, offset int) int {
matched := 0
hasTerminator := false
outer:
for i, r := range input[offset:] {
for _, terminator := range m.runeTerminators {
terminator = unicode.ToLower(terminator)
if m.CaseSensitive {
r = unicode.ToLower(r)
terminator = unicode.ToLower(terminator)
}
if r == terminator {
hasTerminator = true
matched = i
break outer
}
}
}
if !hasTerminator && !m.matchAllIfNoTerminator {
return 0
}
return matched
}
//NewSequenceMatcher creates a new matcher that finds all sequence until find at least one of the provided terminators
func NewSequenceMatcher(terminators ...string) Matcher {
result := &SequenceMatcher{
matchAllIfNoTerminator: true,
Terminators: terminators,
runeTerminators: []rune{},
}
for _, terminator := range terminators {
if len(terminator) != 1 {
result.runeTerminators = []rune{}
break
}
result.runeTerminators = append(result.runeTerminators, rune(terminator[0]))
}
return result
}
//NewTerminatorMatcher creates a new matcher that finds any sequence until find at least one of the provided terminators
func NewTerminatorMatcher(terminators ...string) Matcher {
result := &SequenceMatcher{
Terminators: terminators,
runeTerminators: []rune{},
}
for _, terminator := range terminators {
if len(terminator) != 1 {
result.runeTerminators = []rune{}
break
}
result.runeTerminators = append(result.runeTerminators, rune(terminator[0]))
}
return result
}
//remainingSequenceMatcher represents a matcher that matches all reamining input
type remainingSequenceMatcher struct{}
//Match matches a literal in the input, it returns number of character matched.
func (m *remainingSequenceMatcher) Match(input string, offset int) (matched int) {
return len(input) - offset
}
//Creates a matcher that matches all remaining input
func NewRemainingSequenceMatcher() Matcher {
return &remainingSequenceMatcher{}
}
//CustomIdMatcher represents a matcher that finds any literals with additional custom set of characters in the input
type customIdMatcher struct {
Allowed map[rune]bool
}
func (m *customIdMatcher) isValid(r rune) bool {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
return true
}
return m.Allowed[r]
}
//Match matches a literal in the input, it returns number of character matched.
func (m *customIdMatcher) Match(input string, offset int) int {
var matched = 0
if offset >= len(input) {
return matched
}
for _, r := range input[offset:] {
if !m.isValid(r) {
break
}
matched++
}
return matched
}
//NewCustomIdMatcher creates new custom matcher
func NewCustomIdMatcher(allowedChars ...string) Matcher {
var result = &customIdMatcher{
Allowed: make(map[rune]bool),
}
if len(allowedChars) == 1 && len(allowedChars[0]) > 0 {
for _, allowed := range allowedChars[0] {
result.Allowed[rune(allowed)] = true
}
}
for _, allowed := range allowedChars {
result.Allowed[rune(allowed[0])] = true
}
return result
}
//LiteralMatcher represents a matcher that finds any literals in the input
type BodyMatcher struct {
Begin string
End string
}
//Match matches a literal in the input, it returns number of character matched.
func (m *BodyMatcher) Match(input string, offset int) (matched int) {
beginLen := len(m.Begin)
endLen := len(m.End)
uniEnclosed := m.Begin == m.End
if offset+beginLen >= len(input) {
return 0
}
if input[offset:offset+beginLen] != m.Begin {
return 0
}
var depth = 1
var i = 1
for ; i < len(input)-offset; i++ {
canCheckEnd := offset+i+endLen <= len(input)
if !canCheckEnd {
return 0
}
if !uniEnclosed {
canCheckBegin := offset+i+beginLen <= len(input)
if canCheckBegin {
if string(input[offset+i:offset+i+beginLen]) == m.Begin {
depth++
}
}
}
if string(input[offset+i:offset+i+endLen]) == m.End {
depth--
}
if depth == 0 {
i += endLen
break
}
}
return i
}
//NewBodyMatcher creates a new body matcher
func NewBodyMatcher(begin, end string) Matcher {
return &BodyMatcher{Begin: begin, End: end}
}
// Parses SQL Begin End blocks
func NewBlockMatcher(caseSensitive bool, sequenceStart string, sequenceTerminator string, nestedSequences []string, ignoredTerminators []string) Matcher {
return &BlockMatcher{
CaseSensitive: caseSensitive,
SequenceStart: sequenceStart,
SequenceTerminator: sequenceTerminator,
NestedSequences: nestedSequences,
IgnoredTerminators: ignoredTerminators,
}
}
type BlockMatcher struct {
CaseSensitive bool
SequenceStart string
SequenceTerminator string
NestedSequences []string
IgnoredTerminators []string
}
func (m *BlockMatcher) Match(input string, offset int) (matched int) {
sequenceStart := m.SequenceStart
terminator := m.SequenceTerminator
nestedSequences := m.NestedSequences
ignoredTerminators := m.IgnoredTerminators
in := input
starterLen := len(sequenceStart)
terminatorLen := len(terminator)
if !m.CaseSensitive {
sequenceStart = strings.ToLower(sequenceStart)
terminator = strings.ToLower(terminator)
for i, seq := range nestedSequences {
nestedSequences[i] = strings.ToLower(seq)
}
for i, term := range ignoredTerminators {
ignoredTerminators[i] = strings.ToLower(term)
}
in = strings.ToLower(input)
}
if offset+starterLen >= len(in) {
return 0
}
if in[offset:offset+starterLen] != sequenceStart {
return 0
}
var depth = 1
var i = 1
for ; i < len(in)-offset; i++ {
canCheckEnd := offset+i+terminatorLen <= len(in)
if !canCheckEnd {
return 0
}
canCheckBegin := offset+i+starterLen <= len(in)
if canCheckBegin {
beginning := in[offset+i : offset+i+starterLen]
if beginning == sequenceStart {
depth++
} else {
for _, nestedSeq := range nestedSequences {
nestedLen := len(nestedSeq)
if offset+i+nestedLen >= len(in) {
continue
}
beginning := in[offset+i : offset+i+nestedLen]
if beginning == nestedSeq {
depth++
break
}
}
}
}
ignored := false
for _, ignoredTerm := range ignoredTerminators {
termLen := len(ignoredTerm)
if offset+i+termLen >= len(in) {
continue
}
ending := in[offset+i : offset+i+termLen]
if ending == ignoredTerm {
ignored = true
break
}
}
if !ignored && in[offset+i:offset+i+terminatorLen] == terminator && unicode.IsSpace(rune(in[offset+i-1])) {
depth--
}
if depth == 0 {
i += terminatorLen
break
}
}
return i
}
//KeywordMatcher represents a keyword matcher
type KeywordMatcher struct {
Keyword string
CaseSensitive bool
}
//Match matches keyword in the input, it returns number of character matched.
func (m KeywordMatcher) Match(input string, offset int) (matched int) {
if !(offset+len(m.Keyword)-1 < len(input)) {
return 0
}
if m.CaseSensitive {
if input[offset:offset+len(m.Keyword)] == m.Keyword {
return len(m.Keyword)
}
} else {
if strings.ToLower(input[offset:offset+len(m.Keyword)]) == strings.ToLower(m.Keyword) {
return len(m.Keyword)
}
}
return 0
}
//KeywordsMatcher represents a matcher that finds any of specified keywords in the input
type KeywordsMatcher struct {
Keywords []string
CaseSensitive bool
}
//Match matches any specified keyword, it returns number of character matched.
func (m KeywordsMatcher) Match(input string, offset int) (matched int) {
for _, keyword := range m.Keywords {
if len(input)-offset < len(keyword) {
continue
}
if m.CaseSensitive {
if input[offset:offset+len(keyword)] == keyword {
return len(keyword)
}
} else {
if strings.ToLower(input[offset:offset+len(keyword)]) == strings.ToLower(keyword) {
return len(keyword)
}
}
}
return 0
}
//NewKeywordsMatcher returns a matcher for supplied keywords
func NewKeywordsMatcher(caseSensitive bool, keywords ...string) Matcher {
return &KeywordsMatcher{CaseSensitive: caseSensitive, Keywords: keywords}
}
//IllegalTokenError represents illegal token error
type IllegalTokenError struct {
Illegal *Token
Message string
Expected []int
Position int
}
func (e *IllegalTokenError) Error() string {
return fmt.Sprintf("%v; illegal token at %v [%v], expected %v, but had: %v", e.Message, e.Position, e.Illegal.Matched, e.Expected, e.Illegal.Token)
}
//NewIllegalTokenError create a new illegal token error
func NewIllegalTokenError(message string, expected []int, position int, found *Token) error {
return &IllegalTokenError{
Message: message,
Illegal: found,
Expected: expected,
Position: position,
}
}
//ExpectTokenOptionallyFollowedBy returns second matched token or error if first and second group was not matched
func ExpectTokenOptionallyFollowedBy(tokenizer *Tokenizer, first int, errorMessage string, second ...int) (*Token, error) {
_, _ = ExpectToken(tokenizer, "", first)
return ExpectToken(tokenizer, errorMessage, second...)
}
//ExpectToken returns the matched token or error
func ExpectToken(tokenizer *Tokenizer, errorMessage string, candidates ...int) (*Token, error) {
token := tokenizer.Nexts(candidates...)
hasMatch := HasSliceAnyElements(candidates, token.Token)
if !hasMatch {
return nil, NewIllegalTokenError(errorMessage, candidates, tokenizer.Index, token)
}
return token, nil
}
toolbox-0.33.2/tokenizer_test.go 0000664 0000000 0000000 00000010300 13741102511 0016652 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
)
func TestNewTokenizer(t *testing.T) {
tokenizer := toolbox.NewTokenizer("Z Abcf",
0,
-1,
map[int]toolbox.Matcher{
101: toolbox.KeywordMatcher{"Abc", true},
201: toolbox.CharactersMatcher{Chars: " \n\t"},
102: toolbox.LiteralMatcher{},
},
)
assert.Equal(t, 102, tokenizer.Nexts(101, 201, 102).Token)
assert.Equal(t, 201, tokenizer.Nexts(101, 201, 102).Token)
assert.Equal(t, 101, tokenizer.Nexts(101, 201, 102).Token)
assert.Equal(t, 102, tokenizer.Nexts(101, 201, 102).Token)
}
func Test_NewCustomIdMatcher(t *testing.T) {
{
matcher := toolbox.NewCustomIdMatcher("$")
assert.Equal(t, 5, matcher.Match("Z $Abcf", 2))
assert.Equal(t, 1, matcher.Match("Z Abcf", 0))
assert.Equal(t, 0, matcher.Match("### ##", 0))
}
matcher := toolbox.NewCustomIdMatcher("_", "(", ")")
assert.Equal(t, 1, matcher.Match(" v_sc()", 6))
}
func Test_NewSequenceMatcher(t *testing.T) {
matcher := toolbox.NewSequenceMatcher("&&", "||")
assert.Equal(t, 2, matcher.Match("123", 1))
assert.Equal(t, 4, matcher.Match("123 && 123", 0))
}
func Test_NewSingleSequenceMatcher(t *testing.T) {
matcher := toolbox.NewSequenceMatcher("&")
assert.Equal(t, 0, matcher.Match("123", 1))
assert.Equal(t, 5, matcher.Match("12345&3", 0))
}
func TestMatchKeyword(t *testing.T) {
matcher := toolbox.KeywordMatcher{"Abc", true}
assert.Equal(t, 3, matcher.Match("Z Abcf", 2))
assert.Equal(t, 0, matcher.Match("Z Abcf", 0))
assert.Equal(t, 3, matcher.Match("Z Abc", 2))
}
func TestMatchWhitespace(t *testing.T) {
matcher := toolbox.CharactersMatcher{Chars: " \n\t"}
assert.Equal(t, 0, matcher.Match("1, 2, 3", 0))
assert.Equal(t, 2, matcher.Match("1, \t2, 3", 2))
}
func TestLiteralMatcher(t *testing.T) {
matcher := toolbox.LiteralMatcher{}
assert.Equal(t, 0, matcher.Match(" abc ", 0))
assert.Equal(t, 4, matcher.Match(" a1bc", 1))
}
func TestEOFMatcher(t *testing.T) {
matcher := toolbox.EOFMatcher{}
assert.Equal(t, 0, matcher.Match(" abc ", 0))
assert.Equal(t, 1, matcher.Match(" a1bc", 4))
}
func TestKeywordsMatcher(t *testing.T) {
{
matcher := toolbox.KeywordsMatcher{Keywords: []string{"ab", "xy"},
CaseSensitive: false}
assert.Equal(t, 2, matcher.Match(" abcde", 1))
assert.Equal(t, 0, matcher.Match(" abcde", 0))
}
{
matcher := toolbox.KeywordsMatcher{Keywords: []string{"AB", "xy"},
CaseSensitive: true}
assert.Equal(t, 2, matcher.Match(" ABcde", 1))
assert.Equal(t, 0, matcher.Match("abcde", 0))
}
}
func TestBodyMatcher(t *testing.T) {
{
matcher := toolbox.BodyMatcher{Begin: "{", End: "}"}
var text = " { { \n} } "
pos := matcher.Match(text, 1)
assert.Equal(t, 16, pos)
}
{
matcher := toolbox.BodyMatcher{Begin: "begin", End: "end"}
var text = " begin { \n} end "
pos := matcher.Match(text, 1)
assert.Equal(t, 20, pos)
}
}
func TestBlockMatcher(t *testing.T) {
{
matcher := toolbox.NewBlockMatcher(false, "begin", "end;", []string{"CASE"}, []string{"END IF"})
text := ` TRIGGER users_before_insert
BEFORE INSERT ON users
FOR EACH ROW
BEGIN
SELECT users_seq.NEXTVAL
INTO :new.id
FROM dual;
END;
INSERT INTO DUMMY(ID, NAME) VALUES(2, 'xyz');
`
matcher.Match(text, 65)
}
{
matcher := toolbox.BlockMatcher{
CaseSensitive: false,
SequenceStart: "begin",
SequenceTerminator: "end",
NestedSequences: []string{"case"},
IgnoredTerminators: []string{"end if"},
}
text := "\n\n" +
"BEgin\n" +
"IF get_version()=20\n" +
"select *\n" +
"from table\n" +
"where color = case inventory when 1 then 'brown' when 2 then 'red' END;\n" +
"END IF\n" +
"END;"
pos := matcher.Match(text, 2)
assert.Equal(t, 128, pos)
}
{
matcher := toolbox.BlockMatcher{
CaseSensitive: false,
SequenceStart: "begin",
SequenceTerminator: "end;",
NestedSequences: []string{"case"},
}
text := "bEgIn case 123deabc then 22 End; End;"
pos := matcher.Match(text, 0)
assert.Equal(t, 37, pos)
matcher.CaseSensitive = true
pos = matcher.Match(text, 0)
assert.Equal(t, 0, pos)
matcher.SequenceTerminator = "End;"
matcher.SequenceStart = "bEgIn"
pos = matcher.Match(text, 0)
assert.Equal(t, 37, pos)
}
}
toolbox-0.33.2/types.go 0000664 0000000 0000000 00000014315 13741102511 0014757 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"reflect"
"time"
)
//Zeroable represents object that can call IsZero
type Zeroable interface {
//IsZero returns true, if value of object was zeroed.
IsZero() bool
}
//IsInt returns true if input is an int
func IsInt(input interface{}) bool {
switch input.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
return true
}
return false
}
//IsNumber returns true if type is either float or int
func IsNumber(input interface{}) bool {
return IsFloat(input) || IsInt(input)
}
//IsFloat returns true if input is a float
func IsFloat(input interface{}) bool {
switch input.(type) {
case float32, float64:
return true
}
return false
}
//IsBool returns true if input is a boolean
func IsBool(input interface{}) bool {
switch input.(type) {
case bool:
return true
}
return false
}
//IsString returns true if input is a string
func IsString(input interface{}) bool {
switch input.(type) {
case string:
return true
}
return false
}
//CanConvertToString checks if input can be converted to string
func CanConvertToString(input interface{}) bool {
return reflect.TypeOf(input).AssignableTo(reflect.TypeOf(""))
}
//IsTime returns true if input is a time
func IsTime(input interface{}) bool {
switch input.(type) {
case time.Time:
return true
case *time.Time:
return true
}
return false
}
//IsMap returns true if input is a map
func IsMap(input interface{}) bool {
switch input.(type) {
case map[string]interface{}:
return true
}
candidateType := DereferenceType(reflect.TypeOf(input))
return candidateType.Kind() == reflect.Map
}
//IsStruct returns true if input is a map
func IsStruct(input interface{}) bool {
if input == nil {
return false
}
inputType := DereferenceType(input)
return inputType.Kind() == reflect.Struct
}
//IsSlice returns true if input is a map
func IsSlice(input interface{}) bool {
switch input.(type) {
case []interface{}:
return true
case []string:
return true
}
candidateType := DereferenceType(reflect.TypeOf(input))
return candidateType.Kind() == reflect.Slice
}
//IsFunc returns true if input is a funct
func IsFunc(input interface{}) bool {
candidateType := DereferenceType(reflect.TypeOf(input))
return candidateType.Kind() == reflect.Func
}
//IsZero returns true if input is a zeroable
func IsZero(input interface{}) bool {
if zeroable, ok := input.(Zeroable); ok {
return zeroable.IsZero()
}
return false
}
//IsPointer returns true if input is a pointer
func IsPointer(input interface{}) bool {
if reflectType, ok := input.(reflect.Type); ok {
return reflectType.Kind() == reflect.Ptr
}
return reflect.TypeOf(input).Kind() == reflect.Ptr
}
//AssertPointerKind checks if input is a pointer of the passed in kind, if not it panic with message including name
func AssertPointerKind(input interface{}, kind reflect.Kind, name string) {
AssertTypeKind(reflect.TypeOf(input), reflect.Ptr, name)
AssertTypeKind(reflect.TypeOf(input).Elem(), kind, name)
}
//AssertKind checks if input is of the passed in kind, if not it panic with message including name
func AssertKind(input interface{}, kind reflect.Kind, name string) {
AssertTypeKind(reflect.TypeOf(input), kind, name)
}
//AssertTypeKind checks if dataType is of the passed in kind, if not it panic with message including name
func AssertTypeKind(dataType reflect.Type, kind reflect.Kind, name string) {
if dataType.Kind() != kind {
panic(fmt.Sprintf("failed to check: %v - expected kind: %v but found %v (%v)", name, kind.String(), dataType.Kind(), dataType.String()))
}
}
//DiscoverValueByKind returns unwrapped input that matches expected kind, or panic if this is not possible
func DiscoverValueByKind(input interface{}, expected reflect.Kind) reflect.Value {
result, err := TryDiscoverValueByKind(input, expected)
if err == nil {
return result
}
panic(err)
}
//TryDiscoverValueByKind returns unwrapped input that matches expected kind, or panic if this is not possible
func TryDiscoverValueByKind(input interface{}, expected reflect.Kind) (reflect.Value, error) {
value, ok := input.(reflect.Value)
if !ok {
value = reflect.ValueOf(input)
}
if value.Kind() == expected {
return value, nil
} else if value.Kind() == reflect.Ptr {
return TryDiscoverValueByKind(value.Elem(), expected)
} else if value.Kind() == reflect.Interface {
return TryDiscoverValueByKind(value.Elem(), expected)
}
return value, fmt.Errorf("failed to discover value by kind expected: %v, actual:%T on %v:", expected.String(), value.Type(), value)
}
//IsValueOfKind returns true if passed in input is of supplied kind.
func IsValueOfKind(input interface{}, kind reflect.Kind) bool {
value, ok := input.(reflect.Value)
if !ok {
value = reflect.ValueOf(input)
}
if value.Kind() == kind {
return true
} else if value.Kind() == reflect.Ptr {
return IsValueOfKind(value.Elem(), kind)
} else if value.Kind() == reflect.Interface {
return IsValueOfKind(value.Elem(), kind)
}
return false
}
//DiscoverTypeByKind returns unwrapped input type that matches expected kind, or panic if this is not possible
func DiscoverTypeByKind(input interface{}, expected reflect.Kind) reflect.Type {
result, err := TryDiscoverTypeByKind(input, expected)
if err != nil {
panic(err)
}
return result
}
//TryDiscoverTypeByKind returns unwrapped input type that matches expected kind, or error
func TryDiscoverTypeByKind(input interface{}, expected reflect.Kind) (reflect.Type, error) {
value, ok := input.(reflect.Type)
if !ok {
value = reflect.TypeOf(input)
}
if value.Kind() == expected {
return value, nil
} else if value.Kind() == reflect.Ptr || value.Kind() == reflect.Slice {
return TryDiscoverTypeByKind(value.Elem(), expected)
}
return nil, fmt.Errorf("failed to discover type by kind %v, on %v:", expected.String(), value)
}
//DiscoverComponentType returns type unwrapped from pointer, slice or map
func DiscoverComponentType(input interface{}) reflect.Type {
valueType, ok := input.(reflect.Type)
if !ok {
valueType = reflect.TypeOf(input)
}
if valueType.Kind() == reflect.Ptr {
return DiscoverComponentType(valueType.Elem())
} else if valueType.Kind() == reflect.Slice {
return valueType.Elem()
} else if valueType.Kind() == reflect.Map {
return valueType.Elem()
}
return valueType
}
toolbox-0.33.2/types_test.go 0000664 0000000 0000000 00000005303 13741102511 0016013 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"fmt"
"reflect"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
)
type User2 struct {
Name string `column:"name"`
DateOfBirth time.Time `column:"date" dateLayout:"2006-01-02 15:04:05.000000"`
Id int `autoincrement:"true"`
Other string `transient:"true"`
}
func TestAssertKind(t *testing.T) {
toolbox.AssertKind(User2{}, reflect.Struct, "user")
toolbox.AssertKind((*User2)(nil), reflect.Ptr, "user")
defer func() {
if err := recover(); err != nil {
expected := "failed to check: User - expected kind: ptr but found struct (toolbox_test.User2)"
actual := fmt.Sprintf("%v", err)
assert.Equal(t, actual, expected, "Assert Kind")
}
}()
toolbox.AssertKind(User2{}, reflect.Ptr, "User")
}
func TestAssertPointerKind(test *testing.T) {
toolbox.AssertPointerKind(&User2{}, reflect.Struct, "user")
toolbox.AssertPointerKind((*User2)(nil), reflect.Struct, "user")
}
func TestTypeDetection(t *testing.T) {
assert.False(t, toolbox.IsFloat(3))
assert.True(t, toolbox.IsFloat(3.0))
assert.True(t, toolbox.CanConvertToFloat(3.0))
assert.True(t, toolbox.CanConvertToFloat("3"))
assert.True(t, toolbox.CanConvertToFloat(3))
assert.False(t, toolbox.CanConvertToFloat(false))
assert.False(t, toolbox.IsInt(3.0))
assert.True(t, toolbox.IsInt(3))
assert.True(t, toolbox.CanConvertToInt(3))
assert.True(t, toolbox.CanConvertToInt("3"))
assert.False(t, toolbox.CanConvertToInt(true))
assert.False(t, toolbox.CanConvertToInt(3.3))
assert.False(t, toolbox.IsBool(3.0))
assert.True(t, toolbox.IsBool(true))
assert.False(t, toolbox.IsString(3.0))
assert.True(t, toolbox.IsString("abc"))
assert.True(t, toolbox.CanConvertToString("abc"))
assert.False(t, toolbox.CanConvertToString(3.2))
assert.False(t, toolbox.IsTime(3.0))
assert.True(t, toolbox.IsTime(time.Now()))
var timeValues = make([]time.Time, 1)
assert.True(t, toolbox.IsZero(timeValues[0]))
assert.False(t, toolbox.IsZero(time.Now()))
assert.False(t, toolbox.IsZero(""))
aString := ""
assert.True(t, toolbox.IsPointer(&aString))
assert.False(t, toolbox.IsPointer(reflect.TypeOf(aString)))
assert.True(t, toolbox.IsPointer(&aString))
}
func TestIsValueOfKind(t *testing.T) {
text := ""
assert.True(t, toolbox.IsValueOfKind(&text, reflect.Ptr))
assert.False(t, toolbox.IsValueOfKind(&text, reflect.Struct))
assert.True(t, toolbox.IsValueOfKind(&text, reflect.String))
values := make([]interface{}, 1)
values[0] = 1
assert.True(t, toolbox.IsValueOfKind(&values[0], reflect.Int))
}
func TestIsFunc(t *testing.T) {
var f = func() {}
assert.True(t, toolbox.IsFunc(&f))
assert.True(t, toolbox.IsFunc(f))
assert.False(t, toolbox.IsFunc(""))
}
toolbox-0.33.2/uri.go 0000664 0000000 0000000 00000007423 13741102511 0014414 0 ustar 00root root 0000000 0000000 package toolbox
import (
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
)
//ExtractURIParameters parses URIs to extract {} defined in templateURI from requestURI, it returns extracted parameters and flag if requestURI matched templateURI
func ExtractURIParameters(templateURI, requestURI string) (map[string]string, bool) {
var expectingValue, expectingName bool
var name, value string
var uriParameters = make(map[string]string)
maxLength := len(templateURI) + len(requestURI)
var requestURIIndex, templateURIIndex int
questionMarkPosition := strings.Index(requestURI, "?")
if questionMarkPosition != -1 {
requestURI = string(requestURI[:questionMarkPosition])
}
for k := 0; k < maxLength; k++ {
var requestChar, routingChar string
if requestURIIndex < len(requestURI) {
requestChar = requestURI[requestURIIndex : requestURIIndex+1]
}
if templateURIIndex < len(templateURI) {
routingChar = templateURI[templateURIIndex : templateURIIndex+1]
}
if (!expectingValue && !expectingName) && requestChar == routingChar && routingChar != "" {
requestURIIndex++
templateURIIndex++
continue
}
if routingChar == "}" {
expectingName = false
templateURIIndex++
}
if expectingValue && requestChar == "/" {
expectingValue = false
}
if expectingName && templateURIIndex < len(templateURI) {
name += routingChar
templateURIIndex++
}
if routingChar == "{" {
expectingValue = true
expectingName = true
templateURIIndex++
}
if expectingValue && requestURIIndex < len(requestURI) {
value += requestChar
requestURIIndex++
}
if !expectingValue && !expectingName && len(name) > 0 {
uriParameters[name] = value
name = ""
value = ""
}
}
if len(name) > 0 && len(value) > 0 {
uriParameters[name] = value
}
matched := requestURIIndex == len(requestURI) && templateURIIndex == len(templateURI)
return uriParameters, matched
}
//URLStripPath removes path from URL
func URLStripPath(URL string) string {
protoIndex := strings.Index(URL, "://")
if protoIndex != -1 {
pathIndex := strings.Index(string(URL[protoIndex+3:]), "/")
if pathIndex != -1 {
return string(URL[:protoIndex+3+pathIndex])
}
}
return URL
}
//URLPathJoin joins URL paths
func URLPathJoin(baseURL, path string) string {
if path == "" {
return baseURL
}
if strings.HasPrefix(path, "/") {
return URLStripPath(baseURL) + path
}
if !strings.HasSuffix(baseURL, "/") {
baseURL += "/"
}
return baseURL + path
}
//URLBase returns base URL
func URLBase(URL string) string {
parsedURL, err := url.Parse(URL)
if err != nil || parsedURL.Path == "" {
return URL
}
pathPosition := strings.Index(URL, parsedURL.Path)
if pathPosition == -1 {
return URL
}
return string(URL[:pathPosition])
}
//URLSplit returns URL with parent path and resource name
func URLSplit(URL string) (string, string) {
parsedURL, err := url.Parse(URL)
if err != nil || parsedURL.Path == "" {
return URL, ""
}
splitPosition := strings.LastIndex(parsedURL.Path, "/")
if splitPosition == -1 {
return URL, ""
}
return fmt.Sprintf("%v%v", URLBase(URL), string(parsedURL.Path[:splitPosition])), string(parsedURL.Path[splitPosition+1:])
}
//Filename reformat file name
func Filename(filename string) string {
if strings.Contains(filename, ":/") {
if parsed, err := url.Parse(filename); err == nil {
filename = parsed.Path
}
}
var root = make([]string, 0)
if strings.HasPrefix(filename, "/") {
root = append(root, "/")
}
elements := append(root, strings.Split(filename, "/")...)
filename = filepath.Join(elements...)
return filename
}
//OpenFile open file converting path to elements and rebuling path safety with path.Join
func OpenFile(filename string) (*os.File, error) {
var file = Filename(filename)
var result, err = os.Open(file)
return result, err
}
toolbox-0.33.2/uri_helper.go 0000664 0000000 0000000 00000002263 13741102511 0015750 0 ustar 00root root 0000000 0000000 package toolbox
import (
"net/url"
"path"
)
//FileSchema file://
var FileSchema = "file://"
//Deprecated start using url.Resource
//ExtractMimeType extracts mime type by extension
func ExtractMimeType(file string) string {
extension := path.Ext(file)
if len(extension) > 1 {
extension = extension[1:]
}
if mimeType, ok := FileExtensionMimeType[extension]; ok {
return mimeType
}
return "text/plain"
}
//QueryIntValue returns query value for passed in url's name or default value
func QueryIntValue(u *url.URL, name string, defaultValue int) int {
value := u.Query().Get(name)
if value == "" {
return defaultValue
}
result := AsInt(value)
if result != 0 {
return result
}
return defaultValue
}
//QueryBoolValue returns query value for passed in url's name or default value
func QueryBoolValue(u *url.URL, name string, defaultValue bool) bool {
value := u.Query().Get(name)
if value == "" {
return defaultValue
}
return AsBoolean(value)
}
//QueryValue returns query value for passed in url's name or default value
func QueryValue(u *url.URL, name, defaultValue string) string {
value := u.Query().Get(name)
if value == "" {
return defaultValue
}
return value
}
toolbox-0.33.2/uri_helper_test.go 0000664 0000000 0000000 00000001150 13741102511 0017001 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"net/url"
"testing"
)
func Test_QueryValue(t *testing.T) {
u, err := url.Parse("http://localhost/?k1=v1&k2=2&k3=false")
assert.Nil(t, err)
assert.Equal(t, "v1", toolbox.QueryValue(u, "k1", "default"))
assert.Equal(t, "default", toolbox.QueryValue(u, "k10", "default"))
assert.Equal(t, 2, toolbox.QueryIntValue(u, "k2", 3))
assert.Equal(t, 3, toolbox.QueryIntValue(u, "k10", 3))
assert.Equal(t, false, toolbox.QueryBoolValue(u, "k3", true))
assert.Equal(t, true, toolbox.QueryBoolValue(u, "k10", true))
}
toolbox-0.33.2/uri_test.go 0000664 0000000 0000000 00000006443 13741102511 0015454 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/url"
"testing"
)
func TestExtractURIParameters(t *testing.T) {
{
parameters, matched := toolbox.ExtractURIParameters("/v1/path/{app}/{version}/", "/v1/path/app/1.0/?v=12")
assert.True(t, matched)
if !matched {
t.FailNow()
}
assert.Equal(t, 2, len(parameters))
assert.Equal(t, "app", parameters["app"])
assert.Equal(t, "1.0", parameters["version"])
}
{
parameters, matched := toolbox.ExtractURIParameters("/v1/path/{ids}/{sub}/a/{name}", "/v1/path/1,2,3,4,5/subpath/a/abc")
assert.True(t, matched)
assert.Equal(t, 3, len(parameters))
assert.Equal(t, "1,2,3,4,5", parameters["ids"])
assert.Equal(t, "subpath", parameters["sub"])
assert.Equal(t, "abc", parameters["name"])
}
{
parameters, matched := toolbox.ExtractURIParameters("/v1/path/{ids}", "/v1/path/this-is-test")
assert.True(t, matched)
assert.Equal(t, 1, len(parameters))
assert.Equal(t, "this-is-test", parameters["ids"])
}
{
_, matched := toolbox.ExtractURIParameters("/v2/path/{ids}/{sub}/a/{name}", "/v1/path/1,2,3,4,5/subpath/a/abc")
assert.False(t, matched)
}
{
_, matched := toolbox.ExtractURIParameters("/v1/path1/{ids}/{sub}/a/{name}", "/v1/path/1,2,3,4,5/subpath/a/abc")
assert.False(t, matched)
}
{
_, matched := toolbox.ExtractURIParameters("/v1/path1/{ids}/{sub}/a/{name}", "/v1/path/1,2,3,4,5/subpath/a/abc")
assert.False(t, matched)
}
{
_, matched := toolbox.ExtractURIParameters("/v1/path/{ids}", "/v1/path/1")
assert.True(t, matched)
}
{
_, matched := toolbox.ExtractURIParameters("/v1/reverse/", "/v1/reverse/")
assert.True(t, matched)
}
{
parameters, matched := toolbox.ExtractURIParameters("/v1/path/{ids}/{sub}/a/{name}", "/v1/path/1,2,3,4,5/subpath/a/abcwrwr")
assert.True(t, matched)
assert.Equal(t, 3, len(parameters))
assert.Equal(t, "1,2,3,4,5", parameters["ids"])
assert.Equal(t, "subpath", parameters["sub"])
assert.Equal(t, "abcwrwr", parameters["name"])
}
}
func TestURLBase(t *testing.T) {
URL := "http://github.com/abc"
baseURL := toolbox.URLBase(URL)
assert.Equal(t, "http://github.com", baseURL)
}
func TestURLSplit(t *testing.T) {
{
URL := "http://github.com/abc/trter/rds"
parentURL, resource := toolbox.URLSplit(URL)
assert.Equal(t, "http://github.com/abc/trter", parentURL)
assert.Equal(t, "rds", resource)
}
}
func TestURLStripPath(t *testing.T) {
{
URL := "http://github.com/abc"
assert.EqualValues(t, "http://github.com", toolbox.URLStripPath(URL))
}
{
URL := "http://github.com"
assert.EqualValues(t, "http://github.com", toolbox.URLStripPath(URL))
}
}
func TestURL_Rename(t *testing.T) {
{
URL := "http://github.com/abc/"
resource := url.NewResource(URL)
resource.Rename("/tmp/abc")
assert.Equal(t, "http://github.com//tmp/abc", resource.URL)
}
}
func TestURLPathJoin(t *testing.T) {
{
URL := "http://github.com/abc"
assert.EqualValues(t, "http://github.com/abc/path/a.txt", toolbox.URLPathJoin(URL, "path/a.txt"))
}
{
URL := "http://github.com/abc/"
assert.EqualValues(t, "http://github.com/abc/path/a.txt", toolbox.URLPathJoin(URL, "path/a.txt"))
}
{
URL := "http://github.com/abc/"
assert.EqualValues(t, "http://github.com/a.txt", toolbox.URLPathJoin(URL, "/a.txt"))
}
}
toolbox-0.33.2/url/ 0000775 0000000 0000000 00000000000 13741102511 0014062 5 ustar 00root root 0000000 0000000 toolbox-0.33.2/url/key.go 0000664 0000000 0000000 00000000272 13741102511 0015202 0 ustar 00root root 0000000 0000000 package url
//AES256Key represents custom key
type AES256Key struct {
Key []byte
Base64Key string
Base64KeyMd5Hash string
Base64KeySha256Hash string
}
toolbox-0.33.2/url/resource.go 0000664 0000000 0000000 00000026136 13741102511 0016250 0 ustar 00root root 0000000 0000000 package url
import (
"bytes"
"encoding/base64"
"fmt"
"github.com/viant/toolbox"
"github.com/viant/toolbox/storage"
"gopkg.in/yaml.v2"
"io/ioutil"
"net/url"
"os"
"path"
"strings"
"time"
)
//Resource represents a URL based resource, with enriched meta info
type Resource struct {
URL string `description:"resource URL or relative or absolute path" required:"true"` //URL of resource
Credentials string `description:"credentials file"` //name of credential file or credential key depending on implementation
ParsedURL *url.URL `json:"-"` //parsed URL resource
Cache string `description:"local cache path"` //Cache path for the resource, if specified resource will be cached in the specified path
CustomKey *AES256Key `description:" content encryption key"`
CacheExpiryMs int //CacheExpiryMs expiry time in ms
modificationTag int64
init string
}
//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
return &Resource{
init: r.init,
URL: r.URL,
Credentials: r.Credentials,
ParsedURL: r.ParsedURL,
Cache: r.Cache,
CacheExpiryMs: r.CacheExpiryMs,
}
}
var defaultSchemePorts = map[string]int{
"ssh": 22,
"scp": 22,
"http": 80,
"https": 443,
}
//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
result := r.ParsedURL.Hostname() + ":" + r.Port()
if r.ParsedURL.User != nil {
result = r.ParsedURL.User.Username() + "@" + result
}
return result
}
//CredentialURL returns url's with provided credential
func (r *Resource) CredentialURL(username, password string) string {
var urlCredential = ""
if username != "" {
urlCredential = username
if password != "" {
urlCredential += ":" + password
}
urlCredential += "@"
}
result := r.ParsedURL.Scheme + "://" + urlCredential + r.ParsedURL.Hostname() + ":" + r.Port() + r.ParsedURL.Path
if r.ParsedURL.RawQuery != "" {
result += "?" + r.ParsedURL.RawQuery
}
return result
}
//Path returns url's path directory, assumption is that directory does not have extension, if path ends with '/' it is being stripped.
func (r *Resource) DirectoryPath() string {
if r.ParsedURL == nil {
return ""
}
var result = r.ParsedURL.Path
parent, name := path.Split(result)
if path.Ext(name) != "" {
result = parent
}
if strings.HasSuffix(result, "/") {
result = string(result[:len(result)-1])
}
return result
}
//Port returns url's port
func (r *Resource) Port() string {
port := r.ParsedURL.Port()
if port == "" && r.ParsedURL != nil {
if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
port = toolbox.AsString(value)
}
}
return port
}
//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
if r == nil {
return nil, fmt.Errorf("Fail to download content on empty resource")
}
if r.Cachable() {
content := r.readFromCache()
if content != nil {
return content, nil
}
}
service, err := storage.NewServiceForURL(r.URL, r.Credentials)
if err != nil {
return nil, err
}
object, err := service.StorageObject(r.URL)
if err != nil {
return nil, err
}
reader, err := service.Download(object)
if err != nil {
return nil, err
}
defer reader.Close()
content, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
}
if r.Cachable() {
_ = ioutil.WriteFile(r.Cache, content, 0666)
}
return content, err
}
//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
var result, err = r.Download()
if err != nil {
return "", err
}
return string(result), err
}
//Decode decodes url's data into target, it support JSON and YAML exp.
func (r *Resource) Decode(target interface{}) (err error) {
defer func() {
if err != nil {
err = fmt.Errorf("failed to decode: %v, %v", r.URL, err)
}
}()
if r.ParsedURL == nil {
if r.ParsedURL, err = url.Parse(r.URL); err != nil {
return err
}
}
ext := path.Ext(r.ParsedURL.Path)
switch ext {
case ".yaml", ".yml":
err = r.YAMLDecode(target)
default:
err = r.JSONDecode(target)
}
return err
}
//DecoderFactory returns new decoder factory for resource
func (r *Resource) DecoderFactory() toolbox.DecoderFactory {
ext := path.Ext(r.ParsedURL.Path)
switch ext {
case ".yaml", ".yml":
return toolbox.NewYamlDecoderFactory()
default:
return toolbox.NewJSONDecoderFactory()
}
}
//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) DecodeWith(target interface{}, decoderFactory toolbox.DecoderFactory) error {
if r == nil {
return fmt.Errorf("fail to %T decode on empty resource", decoderFactory)
}
if decoderFactory == nil {
return fmt.Errorf("fail to decode %v, decoderFactory was empty", r.URL)
}
var content, err = r.Download()
if err != nil {
return err
}
text := string(content)
if toolbox.IsNewLineDelimitedJSON(text) {
if aSlice, err := toolbox.NewLineDelimitedJSON(text); err == nil {
return toolbox.DefaultConverter.AssignConverted(target, aSlice)
}
}
err = decoderFactory.Create(bytes.NewReader(content)).Decode(target)
if err != nil {
return fmt.Errorf("failed to decode: %v, payload: %s", err, content)
}
return err
}
//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
var _, currentName = toolbox.URLSplit(r.URL)
if currentName == "" && strings.HasSuffix(r.URL, "/") {
_, currentName = toolbox.URLSplit(r.URL[:len(r.URL)-1])
currentName += "/"
}
r.URL = strings.Replace(r.URL, currentName, name, 1)
r.ParsedURL, err = url.Parse(r.URL)
return err
}
//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
return r.DecodeWith(target, toolbox.NewJSONDecoderFactory())
}
//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
if interfacePrt, ok := target.(*interface{}); ok {
var data interface{}
if err := r.DecodeWith(&data, toolbox.NewYamlDecoderFactory()); err != nil {
return err
}
if toolbox.IsSlice(data) {
*interfacePrt = data
return nil
}
}
var mapSlice = yaml.MapSlice{}
if err := r.DecodeWith(&mapSlice, toolbox.NewYamlDecoderFactory()); err != nil {
return err
}
if !toolbox.IsMap(target) {
return toolbox.DefaultConverter.AssignConverted(target, mapSlice)
}
resultMap := toolbox.AsMap(target)
for _, v := range mapSlice {
resultMap[toolbox.AsString(v.Key)] = v.Value
}
return nil
}
func (r *Resource) readFromCache() []byte {
if toolbox.FileExists(r.Cache) {
info, err := os.Stat(r.Cache)
var isExpired = false
if err == nil && r.CacheExpiryMs > 0 {
elapsed := time.Now().Sub(info.ModTime())
isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
}
content, err := ioutil.ReadFile(r.Cache)
if err == nil && !isExpired {
return content
}
}
return nil
}
//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
return r.Cache != ""
}
func computeResourceModificationTag(resource *Resource) (int64, error) {
service, err := storage.NewServiceForURL(resource.URL, resource.Credentials)
if err != nil {
return 0, err
}
object, err := service.StorageObject(resource.URL)
if err != nil {
return 0, err
}
var fileInfo = object.FileInfo()
if object.IsContent() {
return fileInfo.Size() + fileInfo.ModTime().UnixNano(), nil
}
var result int64 = 0
objects, err := service.List(resource.URL)
if err != nil {
return 0, err
}
for _, object := range objects {
objectResource := NewResource(object.URL())
if objectResource.ParsedURL.Path == resource.ParsedURL.Path {
continue
}
modificationTag, err := computeResourceModificationTag(NewResource(object.URL(), resource.Credentials))
if err != nil {
return 0, err
}
result += modificationTag
}
return result, nil
}
func (r *Resource) HasChanged() (changed bool, err error) {
if r.modificationTag == 0 {
r.modificationTag, err = computeResourceModificationTag(r)
return false, err
}
var recentModificationTag int64
recentModificationTag, err = computeResourceModificationTag(r)
if err != nil {
return false, err
}
if recentModificationTag != r.modificationTag {
changed = true
r.modificationTag = recentModificationTag
}
return changed, err
}
func normalizeURL(URL string) string {
if strings.Contains(URL, "://") {
var protoPosition = strings.Index(URL, "://")
if protoPosition != -1 {
var urlSuffix = string(URL[protoPosition+3:])
urlSuffix = strings.Replace(urlSuffix, "//", "/", len(urlSuffix))
URL = string(URL[:protoPosition+3]) + urlSuffix
}
return URL
}
if !strings.HasPrefix(URL, "/") {
currentDirectory, _ := os.Getwd()
if strings.Contains(URL, "..") {
fragments := strings.Split(URL, "/")
var index = 0
var offset = 0
if fragments[0] == "." {
offset = 1
}
for index = offset; index < len(fragments); index++ {
var fragment = fragments[index]
if fragment == ".." {
currentDirectory, _ = path.Split(currentDirectory)
if strings.HasSuffix(currentDirectory, "/") {
currentDirectory = string(currentDirectory[:len(currentDirectory)-1])
}
continue
}
break
}
return toolbox.FileSchema + path.Join(currentDirectory, strings.Join(fragments[index:], "/"))
}
currentDirectory, err := os.Getwd()
if err == nil {
candidate := path.Join(currentDirectory, URL)
URL = candidate
}
}
return toolbox.FileSchema + URL
}
func (r *Resource) Init() (err error) {
if r.init == r.URL {
return nil
}
r.init = r.URL
r.URL = normalizeURL(r.URL)
r.ParsedURL, err = url.Parse(r.URL)
return err
}
//DownloadBase64 loads base64 resource content
func (r *Resource) DownloadBase64() (string, error) {
storageService, err := storage.NewServiceForURL(r.URL, r.Credentials)
if err != nil {
return "", err
}
reader, err := storage.Download(storageService, r.URL)
if err != nil {
return "", err
}
defer func() {
_ = reader.Close()
}()
data, err := ioutil.ReadAll(reader)
if err != nil {
return "", err
}
_, err = base64.StdEncoding.DecodeString(string(data))
if err == nil {
return string(data), nil
}
return base64.StdEncoding.EncodeToString(data), nil
}
//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
if len(Params) == 0 {
return nil
}
var URL = toolbox.AsString(Params[0])
URL = normalizeURL(URL)
var credential string
if len(Params) > 1 {
credential = toolbox.AsString(Params[1])
}
var cache string
if len(Params) > 2 {
cache = toolbox.AsString(Params[2])
}
var cacheExpiryMs int
if len(Params) > 3 {
cacheExpiryMs = toolbox.AsInt(Params[3])
}
parsedURL, _ := url.Parse(URL)
return &Resource{
init: URL,
ParsedURL: parsedURL,
URL: URL,
Credentials: credential,
Cache: cache,
CacheExpiryMs: cacheExpiryMs,
}
}
toolbox-0.33.2/url/resource_test.go 0000664 0000000 0000000 00000011565 13741102511 0017307 0 ustar 00root root 0000000 0000000 package url_test
import (
"fmt"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"github.com/viant/toolbox/url"
"io/ioutil"
"os"
"path"
"strings"
"testing"
)
func TestNewResource(t *testing.T) {
{
var resource = url.NewResource("./../../test")
fmt.Printf("%v\n", resource.ParsedURL.Path)
assert.True(t, strings.HasSuffix(resource.DirectoryPath(), "viant/test"))
}
{
var resource = url.NewResource("https://raw.githubusercontent.com/viant/toolbox/master/LICENSE.txt")
assert.EqualValues(t, resource.ParsedURL.String(), "https://raw.githubusercontent.com/viant/toolbox/master/LICENSE.txt")
//data, err := resource.Download()
//assert.Nil(t, err)
//assert.NotNil(t, data)
}
{
var resource = url.NewResource("https://raw.githubusercontent.com/viant//toolbox//master/LICENSE.txt")
assert.Equal(t, "https://raw.githubusercontent.com/viant/toolbox/master/LICENSE.txt", resource.URL)
}
{
var resource = url.NewResource("./../test")
assert.True(t, strings.HasSuffix(resource.DirectoryPath(), "/toolbox/test"))
}
{
var resource = url.NewResource("../test")
assert.True(t, strings.HasSuffix(resource.DirectoryPath(), "/toolbox/test"))
}
}
func TestNew_CredentialURL(t *testing.T) {
{
var resource = url.NewResource("https://raw.githubusercontent.com:80/viant/toolbox/master/LICENSE.txt?check=1&p=2")
var URL = resource.CredentialURL("smith", "123")
assert.EqualValues(t, "https://smith:123@raw.githubusercontent.com:80/viant/toolbox/master/LICENSE.txt?check=1&p=2", URL)
}
{
var resource = url.NewResource("https://raw.githubusercontent.com:80/viant/toolbox/master/LICENSE.txt")
var URL = resource.CredentialURL("smith", "")
assert.EqualValues(t, "https://smith@raw.githubusercontent.com:80/viant/toolbox/master/LICENSE.txt", URL)
}
}
func TestNew_DirectoryPath(t *testing.T) {
{
var resource = url.NewResource("https://raw.githubusercontent.com:80/viant/toolbox/master/LICENSE.txt")
assert.EqualValues(t, "/viant/toolbox/master", resource.DirectoryPath())
}
{
var resource = url.NewResource("https://raw.githubusercontent.com:80/viant/toolbox/master/avc")
assert.EqualValues(t, "/viant/toolbox/master/avc", resource.DirectoryPath())
}
{
var resource = url.NewResource("hter")
assert.True(t, strings.HasSuffix(resource.DirectoryPath(), "hter"))
}
}
func TestResource_YamlDecode(t *testing.T) {
if os.Getenv("TMPDIR")== "" {
return
}
var filename1 = path.Join(os.Getenv("TMPDIR"), "resource1.yaml")
var filename2 = path.Join(os.Getenv("TMPDIR"), "resource2.yaml")
_ = toolbox.RemoveFileIfExist(filename1, filename2)
var aMap = map[string]interface{}{
"a": 1,
"b": "123",
"c": []int{1, 3, 6},
}
file, err := os.OpenFile(filename1, os.O_CREATE|os.O_RDWR, 0644)
if assert.Nil(t, err) {
err = toolbox.NewYamlEncoderFactory().Create(file).Encode(aMap)
assert.Nil(t, err)
}
{
var resource = url.NewResource(filename1)
assert.EqualValues(t, resource.ParsedURL.String(), toolbox.FileSchema+filename1)
var resourceData = make(map[string]interface{})
err = resource.YAMLDecode(&resourceData)
assert.Nil(t, err)
assert.EqualValues(t, resourceData["a"], 1)
assert.EqualValues(t, resourceData["b"], "123")
}
YAML := `init:
defaultUser: &defaultUser
name: bob
age: 18
pipeline:
test:
init:
users:
<<: *defaultUser
age: 24
action: print
message: I got $users`
err = ioutil.WriteFile(filename2, []byte(YAML), 0644)
assert.Nil(t, err)
{
var resource = url.NewResource(filename2)
var resourceData = make(map[string]interface{})
err = resource.YAMLDecode(&resourceData)
assert.Nil(t, err)
if normalized, err := toolbox.NormalizeKVPairs(resourceData); err == nil {
resourceData = toolbox.AsMap(normalized)
}
//TODO add actual test once yaml reference is patched
//exposes issue with yaml reference
}
}
func TestResource_JsonDecode(t *testing.T) {
if os.Getenv("TMPDIR") == "" {
return
}
var filename = path.Join(os.Getenv("TMPDIR"), "resource.json")
toolbox.RemoveFileIfExist(filename)
defer toolbox.RemoveFileIfExist(filename)
var aMap = map[string]interface{}{
"a": 1,
"b": "123",
}
file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0644)
if assert.Nil(t, err) {
err = toolbox.NewJSONEncoderFactory().Create(file).Encode(aMap)
assert.Nil(t, err)
}
var resource = url.NewResource(filename)
assert.EqualValues(t, resource.ParsedURL.String(), toolbox.FileSchema+filename)
var resourceData = make(map[string]interface{})
err = resource.Decode(&resourceData)
assert.Nil(t, err)
assert.EqualValues(t, resourceData["a"], 1)
assert.EqualValues(t, resourceData["b"], "123")
}
func TestResource_DecoderFactory(t *testing.T) {
{
resource := url.NewResource("abc.yaml")
factory := resource.DecoderFactory()
assert.NotNil(t, factory)
}
{
resource := url.NewResource("abc.json")
factory := resource.DecoderFactory()
assert.NotNil(t, factory)
}
}
toolbox-0.33.2/value_provider.go 0000664 0000000 0000000 00000024721 13741102511 0016643 0 ustar 00root root 0000000 0000000 package toolbox
import (
"bytes"
"fmt"
"github.com/pkg/errors"
"io/ioutil"
"os"
"strings"
"time"
)
//ValueProvider represents a value provider
type ValueProvider interface {
//Get returns a value for passed in context and arguments. Context can be used to manage state.
Get(context Context, arguments ...interface{}) (interface{}, error)
}
//ValueProviderRegistry registry of value providers
type ValueProviderRegistry interface {
Register(name string, valueProvider ValueProvider)
Contains(name string) bool
Names() []string
Get(name string) ValueProvider
}
type valueProviderRegistryImpl struct {
registry map[string](ValueProvider)
}
func (r valueProviderRegistryImpl) Register(name string, valueProvider ValueProvider) {
r.registry[name] = valueProvider
}
func (r valueProviderRegistryImpl) Contains(name string) bool {
_, ok := r.registry[name]
return ok
}
func (r valueProviderRegistryImpl) Get(name string) ValueProvider {
if result, ok := r.registry[name]; ok {
return result
}
panic(fmt.Sprintf("failed to lookup name: %v", name))
}
func (r valueProviderRegistryImpl) Names() []string {
return MapKeysToStringSlice(&r.registry)
}
//NewValueProviderRegistry create new NewValueProviderRegistry
func NewValueProviderRegistry() ValueProviderRegistry {
var result ValueProviderRegistry = &valueProviderRegistryImpl{
registry: make(map[string]ValueProvider),
}
return result
}
type envValueProvider struct{}
func (p envValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
key := arguments[0].(string)
value, found := os.LookupEnv(key)
if found {
return value, nil
}
return nil, fmt.Errorf("failed to lookup %v in env", key)
}
//NewEnvValueProvider returns a provider that returns a value of env variables.
func NewEnvValueProvider() ValueProvider {
var result ValueProvider = &envValueProvider{}
return result
}
type dateOfBirthProvider struct{}
func (p dateOfBirthProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
if len(arguments) < 1 {
return nil, errors.New("expected | [month], [day], [timeformat]")
}
now := time.Now()
age := AsInt(arguments[0])
var month int = int(now.Month())
var day int = now.Day()
var timeFormat = "yyyy-MM-dd"
if len(arguments) >= 2 {
month = AsInt(arguments[1])
}
if len(arguments) >= 3 {
day = AsInt(arguments[2])
}
if len(arguments) >= 4 {
timeFormat = AsString(arguments[3])
}
dateOfBirthText := fmt.Sprintf("%04d-%02d-%02d", now.Year()-age, month, day)
date, err := time.Parse(DateFormatToLayout("yyyy-MM-dd"), dateOfBirthText)
if err != nil {
return nil, err
}
return date.Format(DateFormatToLayout(timeFormat)), nil
}
//NewDateOfBirthValueProvider provider for computing date for supplied expected age, month and day
func NewDateOfBirthrovider() ValueProvider {
return &dateOfBirthProvider{}
}
type castedValueProvider struct{}
func (p castedValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
key := arguments[0].(string)
if len(arguments) < 2 {
return nil, fmt.Errorf("failed to cast to %v due to invalid number of arguments, Wanted 2 but had:%v", key, len(arguments))
}
switch key {
case "time":
if len(arguments) != 3 {
return nil, fmt.Errorf("failed to cast to time due to invalid number of arguments expected 2, but had %v", len(arguments)-1)
}
castedTime, err := ParseTime(AsString(arguments[1]), AsString(arguments[2]))
if err != nil {
return nil, fmt.Errorf("failed to cast to time %v due to %v", AsString(arguments[1]), err)
}
return castedTime, nil
case "int":
return AsInt(arguments[1]), nil
case "int32":
return int32(AsInt(arguments[1])), nil
case "int64":
return int64(AsInt(arguments[1])), nil
case "float32":
return float32(AsFloat(arguments[1])), nil
case "float":
return AsFloat(arguments[1]), nil
case "bool":
return AsBoolean(arguments[1]), nil
case "string":
return AsString(arguments[1]), nil
}
return nil, fmt.Errorf("failed to cast to %v - unsupported type", key)
}
//NewCastedValueProvider return a provider that return casted value type
func NewCastedValueProvider() ValueProvider {
var result ValueProvider = &castedValueProvider{}
return result
}
type currentTimeProvider struct{}
func (p currentTimeProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
return time.Now(), nil
}
//NewCurrentTimeProvider returns a provder that returns time.Now()
func NewCurrentTimeProvider() ValueProvider {
var result ValueProvider = ¤tTimeProvider{}
return result
}
type timeDiffProvider struct{}
func (p timeDiffProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
var resultTime time.Time
var durationDelta time.Duration
var err error
if len(arguments) >= 1 {
var timeValue *time.Time
var timeLiteral = AsString(arguments[0])
if timeValue, err = TimeAt(timeLiteral); err != nil {
if timeValue, err = ToTime(arguments[0], ""); err != nil {
return nil, err
}
}
resultTime = *timeValue
}
if len(arguments) >= 3 {
var val = AsInt(arguments[1])
var timeUnit = strings.ToLower(AsString(arguments[2]))
durationDelta, err = NewDuration(val, timeUnit)
if err != nil {
return nil, err
}
}
var format = ""
if len(arguments) == 4 {
format = AsString(arguments[3])
}
resultTime = resultTime.Add(durationDelta)
switch format {
case "unix":
return int(resultTime.Unix()+resultTime.UnixNano()) / 1000000000, nil
case "timestamp":
return int(resultTime.Unix()+resultTime.UnixNano()) / 1000000, nil
default:
if len(format) > 0 {
return resultTime.Format(DateFormatToLayout(format)), nil
}
}
return resultTime, nil
}
//NewTimeDiffProvider returns a provider that delta, time unit and optionally format
//format as java date format or unix or timestamp
func NewTimeDiffProvider() ValueProvider {
var result ValueProvider = &timeDiffProvider{}
return result
}
type weekdayProvider struct{}
func (p weekdayProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
var now = time.Now()
return int(now.Weekday()), nil
}
func NewWeekdayProvider() ValueProvider {
return &weekdayProvider{}
}
type nilValueProvider struct{}
func (p nilValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
return nil, nil
}
//NewNilValueProvider returns a provider that returns a nil
func NewNilValueProvider() ValueProvider {
var result ValueProvider = &nilValueProvider{}
return result
}
//ConstValueProvider represnet a const value provider
type ConstValueProvider struct {
Value string
}
//Get return provider value
func (p ConstValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
return p.Value, nil
}
//NewConstValueProvider returns a provider that returns a nil
func NewConstValueProvider(value string) ValueProvider {
var result ValueProvider = &ConstValueProvider{Value: value}
return result
}
type currentDateProvider struct{}
func (p currentDateProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
return time.Now().Local().Format("20060102"), nil
}
//NewCurrentDateProvider returns a provider that returns current date in the format yyyymmdd, i.e. 20170205
func NewCurrentDateProvider() ValueProvider {
var result ValueProvider = ¤tDateProvider{}
return result
}
//Dictionary represents simply lookup interface
type Dictionary interface {
//Get returns value for passed in key or error
Get(key string) (interface{}, error)
//Exists checks if key exists
Exists(key string) bool
}
//MapDictionary alias to map of string and interface{}
type MapDictionary map[string]interface{}
func (d *MapDictionary) Get(name string) (interface{}, error) {
if result, found := (*d)[name]; found {
return result, nil
}
return nil, fmt.Errorf("failed to lookup: %v", name)
}
func (d *MapDictionary) Exists(name string) bool {
_, found := (*d)[name]
return found
}
type dictionaryProvider struct {
dictionaryContentKey interface{}
}
func (p dictionaryProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
if len(arguments) == 0 {
return nil, fmt.Errorf("expected at least one argument but had 0")
}
var key = AsString(arguments[0])
var dictionary Dictionary
context.GetInto(p.dictionaryContentKey, &dictionary)
if len(arguments) == 1 && !dictionary.Exists(key) {
return nil, nil
}
return dictionary.Get(key)
}
//NewDictionaryProvider creates a new Dictionary provider, it takes a key context that is a MapDictionary pointer
func NewDictionaryProvider(contextKey interface{}) ValueProvider {
return &dictionaryProvider{contextKey}
}
type betweenPredicateValueProvider struct{}
func (p *betweenPredicateValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
if len(arguments) != 2 {
return nil, fmt.Errorf("expected 2 arguments with between predicate but had %v", len(arguments))
}
predicate := NewBetweenPredicate(arguments[0], arguments[1])
return &predicate, nil
}
//NewBetweenPredicateValueProvider returns a new between value provider
func NewBetweenPredicateValueProvider() ValueProvider {
return &betweenPredicateValueProvider{}
}
type withinSecPredicateValueProvider struct{}
func (p *withinSecPredicateValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
if len(arguments) != 3 {
return nil, fmt.Errorf("expected 3 arguments predicate, but had %v", len(arguments))
}
if arguments[0] == "now" {
arguments[0] = time.Now()
}
dateFormat := AsString(arguments[2])
dateLayout := DateFormatToLayout(dateFormat)
targetTime := AsTime(arguments[0], dateLayout)
if targetTime == nil {
return nil, fmt.Errorf("Unable convert %v to time.Time", arguments[0])
}
delta := AsInt(arguments[1])
predicate := NewWithinPredicate(*targetTime, delta, dateLayout)
return &predicate, nil
}
//NewWithinSecPredicateValueProvider returns a new within second value provider
func NewWithinSecPredicateValueProvider() ValueProvider {
return &withinSecPredicateValueProvider{}
}
type fileValueProvider struct {
trim bool
}
func (p *fileValueProvider) Get(context Context, arguments ...interface{}) (interface{}, error) {
filePath := AsString(arguments[0])
fileContent, err := ioutil.ReadFile(filePath)
if err != nil {
panic(err)
}
if p.trim {
fileContent = bytes.TrimSpace(fileContent)
}
result := string(fileContent)
return result, nil
}
//NewFileValueProvider create new file value provider
func NewFileValueProvider(trim bool) ValueProvider {
return &fileValueProvider{trim: trim}
}
toolbox-0.33.2/value_provider_test.go 0000664 0000000 0000000 00000013341 13741102511 0017676 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
)
func TestNewEnvValueProvider(t *testing.T) {
provider := toolbox.NewEnvValueProvider()
{
_, err := provider.Get(nil, "USER")
assert.Nil(t, err)
}
{
_, err := provider.Get(nil, "_blahblah")
assert.NotNil(t, err)
}
}
func TestNewCastedValueProvider(t *testing.T) {
provider := toolbox.NewCastedValueProvider()
for _, source := range []interface{}{2, "2"} {
value, err := provider.Get(nil, "int", source)
assert.Nil(t, err)
assert.Equal(t, 2, value)
}
{
_, err := provider.Get(nil, "int")
assert.NotNil(t, err, "Invalid number of parameters")
}
for _, source := range []interface{}{2, "2", 2.0} {
value, err := provider.Get(nil, "float", source)
assert.Nil(t, err)
assert.Equal(t, 2.0, value)
}
for _, source := range []interface{}{true, "true", 1} {
value, err := provider.Get(nil, "bool", source)
assert.Nil(t, err)
assert.Equal(t, true, value)
}
for _, source := range []interface{}{1, "1"} {
value, err := provider.Get(nil, "string", source)
assert.Nil(t, err)
assert.Equal(t, "1", value)
}
{
value, err := provider.Get(nil, "time", "2016-02-22 12:32:01 UTC", toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss z"))
assert.Nil(t, err)
timeValue := value.(time.Time)
assert.Equal(t, int64(1456144321), timeValue.Unix())
}
{
_, err := provider.Get(nil, "time", "2016/02-22 12:32:01 UTC", toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss"))
assert.NotNil(t, err, "invalid format")
}
{
_, err := provider.Get(nil, "time", "2016-02-22 12:32:01 UTC", toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss z"), "1")
assert.NotNil(t, err, "to many parameters")
}
{
_, err := provider.Get(nil, "ABC", "1")
assert.NotNil(t, err, "NOT IMPLEMENTED")
}
}
func TestNewWeekdayProvider(t *testing.T) {
provider := toolbox.NewWeekdayProvider()
value, err := provider.Get(toolbox.NewContext(), nil)
assert.Nil(t, err)
assert.NotNil(t, value)
assert.Equal(t, toolbox.AsInt(value), int(time.Now().Weekday()))
}
func TestNewCurrentTimeProvider(t *testing.T) {
provider := toolbox.NewCurrentTimeProvider()
value, err := provider.Get(nil)
assert.Nil(t, err)
assert.NotNil(t, value)
}
func TestNewCurrentDateProvider(t *testing.T) {
provider := toolbox.NewCurrentDateProvider()
value, err := provider.Get(nil)
assert.Nil(t, err)
assert.NotNil(t, value)
}
func TestNewNilProvider(t *testing.T) {
provider := toolbox.NewNilValueProvider()
value, err := provider.Get(nil)
assert.Nil(t, err)
assert.Nil(t, value)
}
func TestNewValueProviderRegistry(t *testing.T) {
registry := toolbox.NewValueProviderRegistry()
assert.False(t, registry.Contains("a"))
registry.Register("a", toolbox.NewNilValueProvider())
assert.True(t, registry.Contains("a"))
provider := registry.Get("a")
assert.NotNil(t, provider)
assert.Equal(t, 1, len(registry.Names()))
}
func TestNewDictionaryProviderRegistry(t *testing.T) {
var dictionary toolbox.MapDictionary = make(map[string]interface{})
var key toolbox.MapDictionary
dictionary["k1"] = "123"
dictionary["k2"] = "xyz"
provider := toolbox.NewDictionaryProvider(&key)
context := toolbox.NewContext()
context.Put(&key, &dictionary)
{
value, err := provider.Get(context, "k1")
assert.Nil(t, err)
assert.Equal(t, "123", value)
}
{
value, err := provider.Get(context, "k2")
assert.Nil(t, err)
assert.Equal(t, "xyz", value)
}
{
value, err := provider.Get(context, "k13", "true")
assert.NotNil(t, err)
assert.Nil(t, value)
}
}
func Test_NewNewTimeProvider(t *testing.T) {
var now = time.Now()
provider := toolbox.NewTimeDiffProvider()
{
result, err := provider.Get(nil, "now", 1, "day")
assert.Nil(t, err)
var timeResult = toolbox.AsTime(result, "")
in23Hours := now.Add(23 * time.Hour)
in25Hours := now.Add(25 * time.Hour)
assert.True(t, timeResult.After(in23Hours))
assert.True(t, timeResult.Before(in25Hours))
}
{
result, err := provider.Get(nil, "now", 1, "hour", "timestamp")
assert.Nil(t, err)
var timeResult = toolbox.AsInt(result)
in59Mins := int(now.Add(59*time.Minute).Unix() * 1000)
in61Mins := int(now.Add(61*time.Minute).Unix() * 1000)
assert.True(t, in59Mins < timeResult)
assert.True(t, timeResult < in61Mins)
}
{
result, err := provider.Get(nil, "now", 1, "week", "unix")
assert.Nil(t, err)
var timeResult = toolbox.AsInt(result)
in6Days := int(now.Add(6 * 24 * time.Hour).Unix())
in8Days := int(now.Add(8 * 24 * time.Hour).Unix())
assert.True(t, in6Days < timeResult)
assert.True(t, timeResult < in8Days)
}
{
result, err := provider.Get(nil, "now", 1, "hour", "h")
assert.Nil(t, err)
assert.Equal(t, time.Now().Hour()%12+1, toolbox.AsInt(result))
}
}
func Test_NewDateOfBirthValueProvider(t *testing.T) {
//provider := toolbox.NewDateOfBirthrovider()
//{
// result, err := provider.Get(toolbox.NewContext(), 3, 6, 3)
// assert.Nil(t, err)
// assert.EqualValues(t, "2016-06-03", toolbox.AsString(result))
//}
//
//{
// result, err := provider.Get(toolbox.NewContext(), 3, 6, 3, "yyyy-MM-dd")
// assert.Nil(t, err)
// assert.EqualValues(t, "2016-06-03", toolbox.AsString(result))
//}
//
//{
// result, err := provider.Get(toolbox.NewContext(), 3, 6, 3, "yyyy")
// assert.Nil(t, err)
// assert.EqualValues(t, "2016", toolbox.AsString(result))
//}
//
//{
// result, err := provider.Get(toolbox.NewContext(), 3, 9, 2, "yyyy-MM")
// assert.Nil(t, err)
// assert.EqualValues(t, "2016-09", toolbox.AsString(result))
//}
//
//{
// result, err := provider.Get(toolbox.NewContext(), 5, 12, 25, "-MM-dd")
// assert.Nil(t, err)
// assert.EqualValues(t, "-12-25", toolbox.AsString(result))
//}
//
//{
// _, err := provider.Get(toolbox.NewContext())
// assert.NotNil(t, err)
//
//}
}
toolbox-0.33.2/waitgroup_helper.go 0000664 0000000 0000000 00000001504 13741102511 0017167 0 ustar 00root root 0000000 0000000 package toolbox
import (
"sync"
"sync/atomic"
"time"
)
// WaitGroup that waits with a timeout
// Returns true if timeout exceeded and false if there was no timeout
func WaitTimeout(wg *sync.WaitGroup, duration time.Duration) bool {
done := make(chan bool, 1)
closed := int32(0)
defer func() {
if atomic.CompareAndSwapInt32(&closed, 0, 1) {
close(done)
}
}()
go func() {
wg.Wait()
if atomic.LoadInt32(&closed) == 0 {
done <- true
}
}()
select {
case <-done: //Wait till the task is complete and channel get unblocked
return false //No durationToken. Normal execution of task completion
case <-time.After(duration): //Wait till durationToken to elapse
//TODO: time.After() creates a timer that does not get GC until timer durationToken gets elapsed. Need to use AfterFunc
return true //Timed out
}
}
toolbox-0.33.2/waitgroup_helper_test.go 0000664 0000000 0000000 00000001330 13741102511 0020223 0 ustar 00root root 0000000 0000000 package toolbox
import (
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestWaitTimeout_TimeoutTriggered(t *testing.T) {
var wg sync.WaitGroup
go sleep(&wg, time.Second)
time.Sleep(100 * time.Millisecond)
isTimeOut := WaitTimeout(&wg, 100 * time.Millisecond)
assert.Equal(t, true, isTimeOut)
}
func TestWaitTimeout_NoTimeout(t *testing.T) {
var wg sync.WaitGroup
go sleep(&wg, 100 * time.Millisecond)
//task will sleep for 3 seconds but timeout is set only for 5 second
isTimeOut := WaitTimeout(&wg, time.Second)
assert.Equal(t, false, isTimeOut)
}
//Method that sleeps for 3 seconds
func sleep(wg *sync.WaitGroup, duration time.Duration) {
wg.Add(1)
time.Sleep(duration)
wg.Done()
}
toolbox-0.33.2/writer_at.go 0000664 0000000 0000000 00000002137 13741102511 0015612 0 ustar 00root root 0000000 0000000 package toolbox
import (
"sync"
)
//ByteWriterAt represents a bytes writer at
type ByteWriterAt struct {
mutex *sync.Mutex
Buffer []byte
position int
}
//WriteAt returns number of written bytes or error
func (w *ByteWriterAt) WriteAt(p []byte, offset int64) (n int, err error) {
w.mutex.Lock()
if int(offset) == w.position {
w.Buffer = append(w.Buffer, p...)
w.position += len(p)
w.mutex.Unlock()
return len(p), nil
} else if w.position < int(offset) {
var diff = (int(offset) - w.position)
var fillingBytes = make([]byte, diff)
w.position += len(fillingBytes)
w.Buffer = append(w.Buffer, fillingBytes...)
w.mutex.Unlock()
return w.WriteAt(p, offset)
} else {
for i := 0; i < len(p); i++ {
var index = int(offset) + i
if index < len(w.Buffer) {
w.Buffer[int(offset)+i] = p[i]
} else {
w.Buffer = append(w.Buffer, p[i:]...)
break
}
}
w.mutex.Unlock()
return len(p), nil
}
}
//NewWriterAt returns a new instance of byte writer at
func NewByteWriterAt() *ByteWriterAt {
return &ByteWriterAt{
mutex: &sync.Mutex{},
Buffer: make([]byte, 0),
}
}
toolbox-0.33.2/writer_at_test.go 0000664 0000000 0000000 00000000530 13741102511 0016644 0 ustar 00root root 0000000 0000000 package toolbox_test
import (
"github.com/stretchr/testify/assert"
"github.com/viant/toolbox"
"testing"
)
func TestWriterAt_WriteAt(t *testing.T) {
writer := toolbox.NewByteWriterAt()
writer.WriteAt([]byte{0x2}, 1)
writer.WriteAt([]byte{0x1}, 0)
writer.WriteAt([]byte{0x3}, 2)
assert.Equal(t, []byte{0x1, 0x02, 0x3}, writer.Buffer)
}
toolbox-0.33.2/yaml.go 0000664 0000000 0000000 00000003202 13741102511 0014546 0 ustar 00root root 0000000 0000000 package toolbox
import (
"bytes"
"fmt"
"gopkg.in/yaml.v2"
)
//AsYamlText converts data structure int text YAML
func AsYamlText(source interface{}) (string, error) {
if IsStruct(source) || IsMap(source) || IsSlice(source) {
buf := new(bytes.Buffer)
err := yaml.NewEncoder(buf).Encode(source)
return buf.String(), err
}
return "", fmt.Errorf("unsupported type: %T", source)
}
//NormalizeKVPairs converts slice of KV paris into a map, and map[interface{}]interface{} to map[string]interface{}
func NormalizeKVPairs(source interface{}) (interface{}, error) {
if source == nil {
return source, nil
}
isDataStruct := IsMap(source) || IsStruct(source) || IsSlice(source)
var err error
var normalized interface{}
if isDataStruct {
var aMap = make(map[string]interface{})
err = ProcessMap(source, func(k, value interface{}) bool {
var key = AsString(k)
aMap[key] = value
if value == nil {
return true
}
if IsMap(value) || IsSlice(value) || IsStruct(value) {
if normalized, err = NormalizeKVPairs(value); err == nil {
aMap[key] = normalized
}
}
return true
})
if err == nil {
return aMap, nil
}
if IsSlice(aMap) {
return source, err
}
if IsSlice(source) { //yaml style map conversion if applicable
aSlice := AsSlice(source)
if len(aSlice) == 0 {
return source, nil
}
for i, item := range aSlice {
if item == nil {
continue
}
if IsMap(item) || IsSlice(item) {
if normalized, err = NormalizeKVPairs(item); err == nil {
aSlice[i] = normalized
} else {
return source, nil
}
}
}
return aSlice, nil
}
}
return source, err
}
toolbox-0.33.2/yaml_test.go 0000664 0000000 0000000 00000003166 13741102511 0015616 0 ustar 00root root 0000000 0000000 package toolbox
import (
"encoding/json"
"strings"
"testing"
"github.com/stretchr/testify/assert"
yaml "gopkg.in/yaml.v2"
)
func TestNormalizeKVPairs(t *testing.T) {
{ //yaml case
YAML := `- Requests:
- URL: http://localhost:5000
Method: GET
Header:
aHeader:
- "v1"
- "v2"
someOtherHeader:
- "CP=RTO"
Body: "hey there"
Cookies:
- Name: aHeader
Value: a-value
DYAMLomain: "localhost"
Expires: "2023-12-16T20:17:38Z"
RawExpires: Sat, 16 Dec 2023 20:17:38 GMT`
var data interface{}
err := yaml.NewDecoder(strings.NewReader(YAML)).Decode(&data)
assert.Nil(t, err)
normalized, err := NormalizeKVPairs(data)
assert.Nil(t, err)
requests := AsMap(AsSlice(normalized)[0])["Requests"]
request := AsMap(AsSlice(requests)[0])
assert.Equal(t, "http://localhost:5000", request["URL"])
header := AsMap(request["Header"])
assert.Equal(t, []interface{}{"v1", "v2"}, header["aHeader"])
}
{
JSON := `[
{"Key":"k1", "Value":"v1"},
{"Key":"k2", "Value":"v2"},
{"Key":"k3", "Value":[
{"Key":"k1", "Value":"v1", "Attr":2}
]}]`
var data interface{}
err := json.NewDecoder(strings.NewReader(JSON)).Decode(&data)
assert.Nil(t, err)
normalized, err := NormalizeKVPairs(data)
assert.Nil(t, err)
aMap := AsMap(normalized)
assert.Equal(t, "v1", aMap["k1"])
assert.Equal(t, "v2", aMap["k2"])
aSlice := AsSlice(aMap["k3"])
assert.NotNil(t, aSlice)
anItem := AsMap(aSlice[0])
assert.Equal(t, "k1", anItem["Key"])
assert.Equal(t, "v1", anItem["Value"])
assert.Equal(t, 2.0, anItem["Attr"])
}
}