stalecucumber-master/0000755000000000000000000000000013011632561013701 5ustar rootrootstalecucumber-master/unpack_test.go0000644000000000000000000003616113011632561016557 0ustar rootrootpackage stalecucumber import "testing" import "strings" import "reflect" import "math/big" import "github.com/hydrogen18/stalecucumber/struct_export_test" func BenchmarkUnpickleInt(b *testing.B) { const protocol2Int = "\x80\x02K*." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2Int)) if err != nil { panic(err) } } } func BenchmarkUnpickleLong(b *testing.B) { const protocol2Long = "\x80\x02\x8a\x01*." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2Long)) if err != nil { panic(err) } } } func BenchmarkUnpickleShortString(b *testing.B) { const protocol2String = "\x80\x02U\x0bHelloPickleq\x00." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2String)) if err != nil { panic(err) } } } func BenchmarkUnpickleLongString(b *testing.B) { const protocol2String = "\x80\x02UnHelloPickleHelloPickleHelloPickleHelloPickleHelloPickleHelloPickleHelloPickleHelloPickleHelloPickleHelloPickleq\x00." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2String)) if err != nil { panic(err) } } } func BenchmarkUnpickleFloat(b *testing.B) { const protocol2Float = "\x80\x02G@E\x00\x00\x00\x00\x00\x00." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2Float)) if err != nil { panic(err) } } } func BenchmarkUnpickleListOfInts(b *testing.B) { const protocol2ListOfInts = "\x80\x02]q\x00(K\x00K\x01K\x02K\x03K\x04K\x05K\x06K\x07K\x08K\te." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2ListOfInts)) if err != nil { panic(err) } } } func BenchmarkUnpickleDict(b *testing.B) { const protocol2Dict = "\x80\x02}q\x00(U\x01aq\x01K\x01U\x01cq\x02U\x05threeq\x03U\x01bq\x04G@\x00\x00\x00\x00\x00\x00\x00U\x01dq\x05\x8a\x01\x04u." for i := 0; i != b.N; i++ { _, err := Unpickle(strings.NewReader(protocol2Dict)) if err != nil { panic(err) } } } func TestUnpackMapIntoStructHiddenField(t *testing.T) { m := make(map[interface{}]interface{}) m["shouldntMessWithMe"] = 2 s := struct_export_test.Struct1{} unpacker := UnpackInto(&s) unpacker.AllowMissingFields = false unpacker.AllowMismatchedFields = false err := unpacker.From(m, nil) if err == nil { t.Fatal("should have failed") } m = make(map[interface{}]interface{}) m["likewise"] = 3 unpacker = UnpackInto(&s) unpacker.AllowMissingFields = false unpacker.AllowMismatchedFields = false err = unpacker.From(m, nil) if err == nil { t.Fatal("should have failed") } } func TestUnpackIntIntoStruct(t *testing.T) { s := struct{}{} err := UnpackInto(&s).From(Unpickle(strings.NewReader("\x80\x02K\x00."))) if err == nil { t.Fatal("Should have failed!") } upe, ok := err.(UnpackingError) if !ok { t.Fatalf("Should have failed with type %T but got %T:%v", upe, err, err) } } const input0AsListOfDicts = "(lp0\n(dp1\nS'a'\np2\nL1L\nsS'c'\np3\nI3\nsS'b'\np4\nI2\nsa(dp5\ng2\nL1L\nsg3\nI3\nsg4\nI4\nsa(dp6\ng2\nL1L\nsg3\nI5\nsg4\nI2\nsa." func TestUnpackListOfDictsIntoSliceOfStructs(t *testing.T) { dst := make([]testStruct, 0) expect := make([]testStruct, 3) expect[0] = testStruct{ A: 1, B: 2, C: 3, } expect[1] = testStruct{ A: 1, B: 4, C: 3, } expect[2] = testStruct{ A: 1, B: 2, C: 5, } err := UnpackInto(&dst).From(Unpickle(strings.NewReader(input0AsListOfDicts))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", dst, expect) } } func TestUnpackListOfDictsIntoSliceOfPointersToStructs(t *testing.T) { dst := make([]*testStruct, 0) expect := make([]*testStruct, 3) expect[0] = &testStruct{ A: 1, B: 2, C: 3, } expect[1] = &testStruct{ A: 1, B: 4, C: 3, } expect[2] = &testStruct{ A: 1, B: 2, C: 5, } err := UnpackInto(&dst).From(Unpickle(strings.NewReader(input0AsListOfDicts))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", dst, expect) } } type testStruct struct { A int64 B int64 C int64 } type testStructWithPointer struct { A int64 B uint64 C *int64 } const input0 = "\x80\x02}q\x00(U\x01aq\x01K\x01U\x01cq\x02K\x03U\x01bq\x03K\x02u." const input0WithLong = "(dp0\nS'a'\np1\nL1L\nsS'c'\np2\nI3\nsS'b'\np3\nI2\ns." func TestUnpackIntoStruct(t *testing.T) { dst := &testStruct{} expect := &testStruct{ A: 1, B: 2, C: 3, } err := UnpackInto(dst).From(Unpickle(strings.NewReader(input0))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } //Test with python long type in input. Generates *big.Int //with value 1 dst = &testStruct{} err = UnpackInto(dst).From(Unpickle(strings.NewReader(input0WithLong))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } } func TestAllowMismatchedFieldsInStruct(t *testing.T) { dest := &testStruct{} //Key "C" has value "Meow" const input = "(dp0\nS'A'\np1\nI1\nsS'C'\np2\nS'Meow'\np3\nsS'B'\np4\nI2\ns." err := UnpackInto(dest).From(Unpickle(strings.NewReader(input))) if err == nil { t.Fatal("Should fail") } unpackErr := err.(UnpackingError) expectErr := ErrTargetTypeMismatch if expectErr != unpackErr.Err { t.Fatalf("Wrong error:%v", err) } expect := &testStruct{A: 1, B: 2, C: 0} dest.C = 133000 unpacker := UnpackInto(dest) unpacker.AllowMismatchedFields = true err = unpacker.From(Unpickle(strings.NewReader(input))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(expect, dest) { t.Fatalf("Got %v expected %v", *dest, *expect) } } func TestUnpackIntoStructWithPointer(t *testing.T) { dst := &testStructWithPointer{} expect := &testStructWithPointer{ A: 1, B: 2, C: new(int64), } *expect.C = 3 err := UnpackInto(dst).From(Unpickle(strings.NewReader(input0))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } //Test again w/ dst.C non-nil dst.A = 0 dst.B = 0 dst.C = new(int64) *dst.C = 1337 err = UnpackInto(dst).From(Unpickle(strings.NewReader(input0))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } //Test again w/ source having {"C": None } const inputCAsNone = "(dp0\nS'A'\np1\nI1\nsS'C'\np2\nNsS'B'\np3\nI2\ns." dst.A = 0 dst.B = 0 err = UnpackInto(dst).From( Unpickle(strings.NewReader(inputCAsNone))) if err != nil { t.Fatal(err) } expect.C = nil expect.A = 1 expect.B = 2 if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } //test with C being a non pointer type dstWithoutPointer := &testStruct{} err = UnpackInto(dstWithoutPointer).From( Unpickle(strings.NewReader(inputCAsNone))) expectedError := UnpackingError{Err: ErrTargetTypeNotPointer, Destination: reflect.ValueOf(&dstWithoutPointer.C), Source: PickleNone{}} if !reflect.DeepEqual(err, expectedError) { t.Fatalf("\n%v\n%v\n", err, expectedError) } //test again w/ C being non pointer type, but allow //mismatched fields dstWithoutPointer.A = 1000 dstWithoutPointer.B = 2000 dstWithoutPointer.C = 3000 unpacker := UnpackInto(dstWithoutPointer) unpacker.AllowMismatchedFields = true err = unpacker.From(Unpickle(strings.NewReader(inputCAsNone))) if err != nil { t.Fatal(err) } expectWithoutPointer := &testStruct{A: 1, B: 2, C: 0} if !reflect.DeepEqual(dstWithoutPointer, expectWithoutPointer) { t.Fatalf("Got %v expected %v", *dstWithoutPointer, *expectWithoutPointer) } //test with C being arbitrarily deep pointer type dstWithManyPointers := &testStructWithDeepPointers{} err = UnpackInto(dstWithManyPointers).From( Unpickle(strings.NewReader(inputCAsNone))) if err != nil { t.Fatal(err) } expectWithManyPointers := &testStructWithDeepPointers{ A: 1, B: 2, C: dstWithManyPointers.C, } if !reflect.DeepEqual(dstWithManyPointers, expectWithManyPointers) { t.Fatalf("Got %v expected %v", *dstWithManyPointers, *expectWithManyPointers) } const EXPECTED_DEPTH = 8 depth := 0 v := reflect.ValueOf(dstWithManyPointers.C) for v.Kind() == reflect.Ptr { depth++ v = v.Elem() } if depth != EXPECTED_DEPTH { t.Fatal("wrong depth") } } type testStructWithDeepPointers struct { A int B int C ********int } const inputB = "\x80\x02}q\x00(U\x01aq\x01K*U\x01cq\x02U\x06foobarq\x03U\x01bq\x04G@*\xbdp\xa3\xd7\n=U\x01eq\x05\x88U\x01dq\x06\x8a\x01\x01u." type testStructB struct { A int B float32 C string D big.Int E bool } type testStructBWithPointers struct { A *int B *float32 C *string D *big.Int E *bool } func TestUnpackStructB(t *testing.T) { dst := &testStructB{} expect := &testStructB{ A: 42, B: 13.37, C: "foobar", D: *big.NewInt(1), E: true, } err := UnpackInto(dst).From(Unpickle(strings.NewReader(inputB))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } dstP := &testStructBWithPointers{} err = UnpackInto(dstP).From(Unpickle(strings.NewReader(inputB))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } } const inputC = "\x80\x02}q\x00(U\x03dogq\x01U\x01aq\x02U\x01bq\x03U\x01cq\x04\x87q\x05U\x05appleq\x06K\x01K\x02K\x03\x87q\x07U\ncanteloupeq\x08h\x05U\x06bananaq\th\x07u." type testStructC struct { Apple []interface{} Banana []interface{} Canteloupe []interface{} Dog []interface{} } func TestUnpackStructC(t *testing.T) { dst := &testStructC{} expect := &testStructC{ Apple: []interface{}{int64(1), int64(2), int64(3)}, Banana: []interface{}{int64(1), int64(2), int64(3)}, Canteloupe: []interface{}{"a", "b", "c"}, Dog: []interface{}{"a", "b", "c"}, } err := UnpackInto(dst).From(Unpickle(strings.NewReader(inputC))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } } const inputD = "\x80\x02}q\x00(U\x08Aardvarkq\x01K\x01U\x05Bolusq\x02G@\x08\x00\x00\x00\x00\x00\x00U\x03Catq\x03}q\x04(U\x05appleq\x05K\x02U\x06bananaq\x06K\x03uu." const inputDWithUnicode = "\x80\x02}q\x00(X\x08\x00\x00\x00Aardvarkq\x01K\x01U\x05Bolusq\x02G@\x08\x00\x00\x00\x00\x00\x00U\x03Catq\x03}q\x04(U\x05appleq\x05K\x02X\x06\x00\x00\x00bananaq\x06K\x03uu." type testStructDWithMap struct { Aardvark uint Bolus float32 Cat map[interface{}]interface{} } type testStructDWithStruct struct { Aardvark uint Bolus float32 Cat struct { Apple int Banana uint } } type testStructDWithTags struct { One uint `pickle:"Aardvark"` Two float32 `pickle:"Bolus"` Three struct { Four int `pickle:"apple"` Five uint `pickle:"banana"` } `pickle:"Cat"` } func TestStructDWithPickleNames(t *testing.T) { dst := &testStructDWithTags{} expect := &testStructDWithTags{ One: 1, Two: 3.0, } expect.Three.Four = 2 expect.Three.Five = 3 err := UnpackInto(dst).From(Unpickle(strings.NewReader(inputD))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } } func TestUnpackStructDWithStruct(t *testing.T) { dst := &testStructDWithStruct{} expect := &testStructDWithStruct{ Aardvark: 1, Bolus: 3.0, } expect.Cat.Apple = 2 expect.Cat.Banana = 3 err := UnpackInto(dst).From(Unpickle(strings.NewReader(inputD))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } dst = &testStructDWithStruct{} err = UnpackInto(dst).From(Unpickle(strings.NewReader(inputDWithUnicode))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } } func TestUnpackStructDWithMap(t *testing.T) { dst := &testStructDWithMap{} expect := &testStructDWithMap{ Aardvark: 1, Bolus: 3.0, Cat: make(map[interface{}]interface{}), } expect.Cat["apple"] = int64(2) expect.Cat["banana"] = int64(3) err := UnpackInto(dst).From(Unpickle(strings.NewReader(inputD))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", *dst, *expect) } } type testStructDWithBadStruct struct { Aardvark uint Bolus float32 Cat struct { Apple string Banana uint } } func TestUnpackStructDWithBadStruct(t *testing.T) { dst := &testStructDWithBadStruct{} err := UnpackInto(dst).From(Unpickle(strings.NewReader(inputD))) if err == nil { t.Fatalf("Should not have unpacked:%v", dst) } } const inputE = "(dp0\nS'ds'\np1\n(lp2\n(dp3\nS'a'\np4\nL1L\nsS'c'\np5\nI3\nsS'b'\np6\nI2\nsa(dp7\ng4\nL1L\nsg5\nI3\nsg6\nI4\nsa(dp8\ng4\nL1L\nsg5\nI5\nsg6\nI2\nsas." type testStructureE struct { Ds []testStruct } func TestUnpackDictWithListOfDictsIntoStructWithListOfDicts(t *testing.T) { dst := testStructureE{} e := testStructureE{} e.Ds = make([]testStruct, 3) e.Ds[0] = testStruct{ A: 1, B: 2, C: 3, } e.Ds[1] = testStruct{ A: 1, B: 4, C: 3, } e.Ds[2] = testStruct{ A: 1, B: 2, C: 5, } err := UnpackInto(&dst).From(Unpickle(strings.NewReader(inputE))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, e) { t.Fatalf("Got %v expected %v", dst, e) } } const inputF = "(lp0\nI0\naI1\naI2\naI3\naI4\na." func TestUnpackSliceOfInts(t *testing.T) { dst := make([]int64, 0) expect := []int64{0, 1, 2, 3, 4} err := UnpackInto(&dst).From(Unpickle(strings.NewReader(inputF))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", dst, expect) } //Test that slices are re used and trimmed when needed for i := range dst { dst[i] = -1 } dst = append(dst, 42) err = UnpackInto(&dst).From(Unpickle(strings.NewReader(inputF))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", dst, expect) } } const inputG = "(lp0\nS'foo'\np1\naVbar\np2\naS'qux'\np3\na." func TestUnpackSliceOfStrings(t *testing.T) { dst := []string{"disappears"} expect := []string{"foo", "bar", "qux"} err := UnpackInto(&dst).From(Unpickle(strings.NewReader(inputG))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", dst, expect) } } const inputH = "(lp0\nS'meow'\np1\naI42\naS'awesome'\np2\na." func TestUnpackHeterogeneousList(t *testing.T) { dst := []interface{}{} expect := []interface{}{"meow", int64(42), "awesome"} err := UnpackInto(&dst).From(Unpickle(strings.NewReader(inputH))) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(dst, expect) { t.Fatalf("Got %v expected %v", dst, expect) } dst2 := []string{} err = UnpackInto(&dst2).From(Unpickle(strings.NewReader(inputH))) if err == nil { t.Fatal(err) } upe, ok := err.(UnpackingError) if !ok { t.Fatalf("Got wrong error type %T:%v", err, err) } i, ok := upe.Source.(int64) if !ok && i == 42 { t.Fatalf("Failed on wrong value %v(%T)", upe.Source, upe.Source) } } func TestUnpackIntInStructIntoBigInt(t *testing.T) { dst := struct { V *big.Int }{} const input = "(dp0\nS'V'\np1\nI1\ns." err := UnpackInto(&dst).From(Unpickle(strings.NewReader(input))) if err != nil { t.Fatal(err) } if dst.V == nil || dst.V.Int64() != 1 { t.Fatal(dst.V) } } stalecucumber-master/python_types.go0000644000000000000000000000077013011632561017001 0ustar rootrootpackage stalecucumber /* This type is used internally to represent a concept known as a mark on the Pickle Machine's stack. Oddly formed pickled data could return this value as the result of Unpickle. In normal usage this type is needed only internally. */ type PickleMark struct{} func (_ PickleMark) String() string { return "PickleMachine Mark" } /* This type is used to represent the Python object "None" */ type PickleNone struct{} func (_ PickleNone) String() string { return "Python None" } stalecucumber-master/unpack.go0000644000000000000000000001715013011632561015515 0ustar rootrootpackage stalecucumber import "reflect" import "fmt" import "errors" import "strings" import "math/big" const PICKLE_TAG = "pickle" type UnpackingError struct { Source interface{} Destination reflect.Value Err error } /* This type is returned when a call to From() fails. Setting "AllowMissingFields" and "AllowMismatchedFields" on the result of "UnpackInto" controls if this error is returned or not. */ func (ue UnpackingError) Error() string { var dv string var dt string k := ue.Destination.Kind() switch k { case reflect.Ptr: dt = fmt.Sprintf("%s", ue.Destination.Type().Elem()) if !ue.Destination.IsNil() { dv = fmt.Sprintf("%v", ue.Destination.Elem().Interface()) } else { dv = "nil" } case reflect.Invalid: dv = "invalid" dt = dv default: dv = fmt.Sprintf("%v", ue.Destination.Interface()) dt = fmt.Sprintf("%s", ue.Destination.Type()) } return fmt.Sprintf("Error unpacking %v(%T) into %s(%s):%v", ue.Source, ue.Source, dv, dt, ue.Err) } var ErrNilPointer = errors.New("Destination cannot be a nil pointer") var ErrNotPointer = errors.New("Destination must be a pointer type") var ErrTargetTypeNotPointer = errors.New("Target type must be a pointer to unpack this value") var ErrTargetTypeOverflow = errors.New("Value overflows target type") var ErrTargetTypeMismatch = errors.New("Target type does not match source type") type unpacker struct { dest reflect.Value AllowMissingFields bool AllowMismatchedFields bool } func UnpackInto(dest interface{}) unpacker { return unpacker{dest: reflect.ValueOf(dest), AllowMissingFields: true, AllowMismatchedFields: false} } func (u unpacker) From(srcI interface{}, err error) error { //Check if an error occurred if err != nil { return err } return u.from(srcI) } func (u unpacker) from(srcI interface{}) error { //Get the value of the destination v := u.dest //The destination must always be a pointer if v.Kind() != reflect.Ptr { return UnpackingError{Source: srcI, Destination: u.dest, Err: ErrNotPointer} } //The destination can never be nil if v.IsNil() { return UnpackingError{Source: srcI, Destination: u.dest, Err: ErrNilPointer} } //Indirect the destination. This gets the actual //value pointed at vIndirect := v for vIndirect.Kind() == reflect.Ptr { if vIndirect.IsNil() { vIndirect.Set(reflect.New(vIndirect.Type().Elem())) } vIndirect = vIndirect.Elem() } //Check the input against known types switch s := srcI.(type) { default: return UnpackingError{Source: srcI, Destination: u.dest, Err: errors.New("Unknown source type")} case PickleNone: vElem := v.Elem() for vElem.Kind() == reflect.Ptr { next := vElem.Elem() if next.Kind() == reflect.Ptr { vElem = next continue } if vElem.CanSet() { vElem.Set(reflect.Zero(vElem.Type())) return nil } } return UnpackingError{Source: srcI, Destination: u.dest, Err: ErrTargetTypeNotPointer} case int64: switch vIndirect.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int64, reflect.Int32: if vIndirect.OverflowInt(s) { return UnpackingError{Source: srcI, Destination: u.dest, Err: ErrTargetTypeOverflow} } vIndirect.SetInt(s) return nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: if s < 0 || vIndirect.OverflowUint(uint64(s)) { return UnpackingError{Source: srcI, Destination: u.dest, Err: ErrTargetTypeOverflow} } vIndirect.SetUint(uint64(s)) return nil } dstBig, ok := vIndirect.Addr().Interface().(*big.Int) if ok { dstBig.SetInt64(s) return nil } case string: switch vIndirect.Kind() { case reflect.String: vIndirect.SetString(s) return nil } case bool: switch vIndirect.Kind() { case reflect.Bool: vIndirect.SetBool(s) return nil } case float64: switch vIndirect.Kind() { case reflect.Float32, reflect.Float64: vIndirect.SetFloat(s) return nil } case *big.Int: dstBig, ok := vIndirect.Addr().Interface().(*big.Int) if ok { dstBig.Set(s) return nil } if vi, err := Int(srcI, nil); err == nil { return unpacker{dest: v, AllowMismatchedFields: u.AllowMismatchedFields, AllowMissingFields: u.AllowMissingFields}.From(vi, nil) } case []interface{}: //Check that the destination is a slice if vIndirect.Kind() != reflect.Slice { return UnpackingError{Source: s, Destination: u.dest, Err: fmt.Errorf("Cannot unpack slice into destination")} } //Check for exact type match if vIndirect.Type().Elem().Kind() == reflect.Interface { vIndirect.Set(reflect.ValueOf(s)) return nil } //Build the value using reflection var replacement reflect.Value if vIndirect.IsNil() || vIndirect.Len() < len(s) { replacement = reflect.MakeSlice(vIndirect.Type(), len(s), len(s)) } else { replacement = vIndirect.Slice(0, len(s)) } for i, srcV := range s { dstV := replacement.Index(i) //Recurse to set the value err := unpacker{dest: dstV.Addr(), AllowMissingFields: u.AllowMissingFields, AllowMismatchedFields: u.AllowMismatchedFields}. From(srcV, nil) if err != nil { return err } } vIndirect.Set(replacement) return nil case map[interface{}]interface{}: //Check to see if the field is exactly //of the type if vIndirect.Kind() == reflect.Map { dstT := vIndirect.Type() if dstT.Key().Kind() == reflect.Interface && dstT.Elem().Kind() == reflect.Interface { vIndirect.Set(reflect.ValueOf(s)) return nil } } var src map[string]interface{} var err error src, err = DictString(srcI, err) if err != nil { return UnpackingError{Source: srcI, Destination: u.dest, Err: fmt.Errorf("Cannot unpack source into struct")} } if vIndirect.Kind() != reflect.Struct { return UnpackingError{Source: src, Destination: u.dest, Err: fmt.Errorf("Cannot unpack into %v", v.Kind().String())} } var fieldByTag map[string]int vIndirectType := reflect.TypeOf(vIndirect.Interface()) numFields := vIndirectType.NumField() for i := 0; i != numFields; i++ { fv := vIndirectType.Field(i) tag := fv.Tag.Get(PICKLE_TAG) if len(tag) != 0 { if fieldByTag == nil { fieldByTag = make(map[string]int) } fieldByTag[tag] = i } } for k, kv := range src { var fv reflect.Value if fieldIndex, ok := fieldByTag[k]; ok { fv = vIndirect.Field(fieldIndex) } else { //Try the name verbatim. This catches //embedded fields as well fv = vIndirect.FieldByName(k) if !fv.IsValid() { //Capitalize the first character. Structs //do not export fields with a lower case //first character capk := strings.ToUpper(k[0:1]) + k[1:] fv = vIndirect.FieldByName(capk) } } if !fv.IsValid() || !fv.CanSet() { if !u.AllowMissingFields { return UnpackingError{Source: src, Destination: u.dest, Err: fmt.Errorf("Cannot find field for key %q", k)} } continue } err := unpacker{dest: fv.Addr(), AllowMismatchedFields: u.AllowMismatchedFields, AllowMissingFields: u.AllowMissingFields}.from(kv) if err != nil { if u.AllowMismatchedFields { if unpackingError, ok := err.(UnpackingError); ok { switch unpackingError.Err { case ErrTargetTypeOverflow, ErrTargetTypeNotPointer, ErrTargetTypeMismatch: fv.Set(reflect.Zero(fv.Type())) continue } } } return err } } return nil } return UnpackingError{Source: srcI, Destination: u.dest, Err: ErrTargetTypeMismatch} } stalecucumber-master/pickle_writer_test.go0000644000000000000000000001545213011632561020141 0ustar rootrootpackage stalecucumber import "testing" import "bytes" import "io" import "reflect" import "math/big" import "github.com/hydrogen18/stalecucumber/struct_export_test" func TestPickleBadTypes(t *testing.T) { c := make(chan int) assertPicklingFails(c, PicklingError{Err: ErrTypeNotPickleable, V: c}, t) } func assertPicklingFails(v interface{}, expect error, t *testing.T) { buf := &bytes.Buffer{} p := NewPickler(buf) _, err := p.Pickle(v) if err == nil { t.Fatalf("Pickling (%T)%v should have failed", v, v) } if !reflect.DeepEqual(err, expect) { t.Fatalf("Expected error (%T)%v got (%T)%v", expect, expect, err, err) } } func TestPickleFloat64(t *testing.T) { roundTrip(1337.42, t) } func TestPickleByte(t *testing.T) { inAndOut(byte(1), int64(1), t) } func TestPickleFloat32(t *testing.T) { var f float32 f = 1337.42 inAndOut(f, float64(f), t) } func TestPickleString(t *testing.T) { roundTrip("test string #1", t) } func TestPickleInt(t *testing.T) { var i int i = 2300000 inAndOut(i, int64(i), t) } func TestPickleInt8(t *testing.T) { var i int8 i = 43 inAndOut(i, int64(i), t) } func TestPickleInt16(t *testing.T) { var i int16 i = 4200 inAndOut(i, int64(i), t) } func TestPickleInt32(t *testing.T) { var i int32 i = 42 inAndOut(i, int64(i), t) } func TestPickleUint(t *testing.T) { var i uint i = 13556263 inAndOut(i, int64(i), t) } func TestPickleUint8(t *testing.T) { var i uint8 i = 22 inAndOut(i, int64(i), t) } func TestPickleUint16(t *testing.T) { var i uint16 i = 10000 inAndOut(i, int64(i), t) } func TestPickleUint32(t *testing.T) { var i uint32 i = 2 inAndOut(i, int64(i), t) i = 4294967295 bi := big.NewInt(int64(i)) inAndOut(i, bi, t) } func TestPickleUint64(t *testing.T) { var i uint64 i = 1580137 inAndOut(i, int64(i), t) i = 18446744073709551615 /** buf := &bytes.Buffer{} p := NewPickler(buf) _, err := p.Pickle(i) if err != nil { t.Fatal(err) } t.Fatalf("%v", buf.Bytes()) **/ bi := big.NewInt(0) bi.SetUint64(i) inAndOut(i, bi, t) var ui uint ui = ^uint(0) bi.SetUint64(uint64(ui)) inAndOut(ui, bi, t) } func TestPickleInt64(t *testing.T) { var i int64 i = 1337 roundTrip(i, t) i = 1 << 48 inAndOut(i, big.NewInt(i), t) i *= -1 inAndOut(i, big.NewInt(i), t) i = 1 << 32 i *= -1 inAndOut(i, big.NewInt(i), t) } func TestPickleSlice(t *testing.T) { data := make([]interface{}, 3) data[0] = "meow" data[1] = int64(1336) data[2] = float64(42.0) roundTrip(data, t) var array [3]int out := make([]interface{}, len(array)) array[0] = 100 array[1] = 200 array[2] = 300 for i, v := range array { out[i] = int64(v) } inAndOut(array, out, t) } func TestPickleMap(t *testing.T) { data := make(map[interface{}]interface{}) data[int64(1)] = int64(10) data[int64(2)] = int64(20) data[int64(3)] = "foobar" data[int64(4)] = float64(4.0) data[int64(5)] = float64(2.0) data["meow"] = "foooooocool" roundTrip(data, t) in := make(map[string]float32) out := make(map[interface{}]interface{}) in["foo"] = 2.0 out["foo"] = 2.0 in["bar"] = 4.0 out["bar"] = 4.0 inAndOut(in, out, t) } type exampleStruct struct { Apple int Banana int32 Cat uint32 Dog int8 Elephant string Fart float32 `pickle:"fart"` Golf uint64 } type exampleEmbeddedStruct struct { exampleStruct Hiking string } func TestRoundTripStruct(t *testing.T) { in := exampleStruct{ 1, 2, 3, 4, "hello world!", 1151356.0, 9223372036854775807, } inAndUnpack(in, t) in2 := struct_export_test.Struct1{} in2.Joker = "meow" in2.Killer = 23.37 in2.Lawnmower = 23 in2.Duplicate = 42.12 inAndUnpack(in2, t) in3 := exampleEmbeddedStruct{ exampleStruct: in, Hiking: "is fun", } inAndUnpack(in3, t) } func TestPickleSnowman(t *testing.T) { roundTrip("This is a snowman: ☃", t) } func TestPicklePointer(t *testing.T) { var myptr *int myptr = new(int) *myptr = 42 inAndUnpack(myptr, t) inAndUnpack(&myptr, t) myptr = nil inAndUnpack(&myptr, t) } func TestPickleStruct(t *testing.T) { example := struct { Apple uint64 Banana float32 C string dog func(int) //Not exported, ignored by pickler }{ 1, 2, "hello pickles", nil, } out := make(map[interface{}]interface{}) out["Apple"] = int64(1) out["Banana"] = float64(2.0) out["C"] = example.C inAndOut(example, out, t) example2 := struct { Elephant int Fart string `pickle:"fart"` Golf float32 }{ 14, "woohoo", 13.37, } out = make(map[interface{}]interface{}) out["Elephant"] = int64(example2.Elephant) out["Golf"] = float64(example2.Golf) out["fart"] = example2.Fart inAndOut(example2, out, t) } func TestPickleBool(t *testing.T) { var b bool roundTrip(b, t) b = true roundTrip(b, t) } func TestPickleBigInt(t *testing.T) { i := big.NewInt(1) i.Lsh(i, 42) roundTrip(i, t) i.SetUint64(1) i.Lsh(i, 256*8) roundTrip(i, t) i.Mul(i, big.NewInt(-1)) roundTrip(i, t) i = big.NewInt(0) roundTrip(i, t) } func TestPickleTuple(t *testing.T) { myTuple := NewTuple(1, 2, "foobar") buf := &bytes.Buffer{} pickler := NewPickler(buf) _, err := pickler.Pickle(myTuple) if err != nil { t.Fatal(err) } out := []interface{}{int64(1), int64(2), "foobar"} //Verify it can be read back inAndOut(myTuple, out, t) myTuple = NewTuple() out = []interface{}{} inAndOut(myTuple, out, t) myTuple = NewTuple("a") out = []interface{}{"a"} inAndOut(myTuple, out, t) myTuple = NewTuple("a", "b") out = []interface{}{"a", "b"} inAndOut(myTuple, out, t) myTuple = NewTuple("a", "b", "c", nil, "d") out = []interface{}{"a", "b", "c", PickleNone{}, "d"} inAndOut(myTuple, out, t) } func inAndUnpack(v interface{}, t *testing.T) { buf := &bytes.Buffer{} p := NewPickler(buf) _, err := p.Pickle(v) if err != nil { t.Fatalf("Failed writing type %T:\n%v", v, err) } w := reflect.New(reflect.TypeOf(v)) err = UnpackInto(w.Interface()).From(Unpickle(bytes.NewReader(buf.Bytes()))) if err != nil { t.Fatalf("Failed unpickling %T:%v", w, err) } wi := reflect.Indirect(w).Interface() if !reflect.DeepEqual(v, wi) { t.Fatalf("\nFrom:%x\n---EXPECTED:(%T)\n%v\n---GOT:(%T)\n%v", buf.Bytes(), v, v, wi, wi) } } func inAndOut(v, w interface{}, t *testing.T) { buf := &bytes.Buffer{} p := NewPickler(buf) _, err := p.Pickle(v) if err != nil { t.Fatalf("Failed writing type %T:%v", v, err) } sanityCheck(buf, t, w) } func roundTrip(v interface{}, t *testing.T) { buf := &bytes.Buffer{} p := NewPickler(buf) _, err := p.Pickle(v) if err != nil { t.Fatalf("Failed writing type %T:%v", v, err) } sanityCheck(buf, t, v) } func sanityCheck(r io.Reader, t *testing.T, expect interface{}) { v, err := Unpickle(r) if err != nil { t.Fatalf("Failed to unpickle own output:%v", err) } if !reflect.DeepEqual(v, expect) { t.Fatalf("\n---EXPECTED:(%T)\n%v\n---GOT:(%T)\n%v", expect, expect, v, v) } } stalecucumber-master/protocol_1.go0000644000000000000000000002757613011632561016332 0ustar rootrootpackage stalecucumber import "fmt" import "errors" /** Opcode: BININT (0x4a) Push a four-byte signed integer. This handles the full range of Python (short) integers on a 32-bit box, directly as binary bytes (1 for the opcode and 4 for the integer). If the integer is non-negative and fits in 1 or 2 bytes, pickling via BININT1 or BININT2 saves space. ** Stack before: [] Stack after: [int] **/ func (pm *PickleMachine) opcode_BININT() error { var v int32 err := pm.readBinaryInto(&v, false) if err != nil { return err } pm.push(int64(v)) return nil } /** Opcode: BININT1 (0x4b) Push a one-byte unsigned integer. This is a space optimization for pickling very small non-negative ints, in range(256). ** Stack before: [] Stack after: [int] **/ func (pm *PickleMachine) opcode_BININT1() error { var v uint8 err := pm.readBinaryInto(&v, false) if err != nil { return err } pm.push(int64(v)) return nil } /** Opcode: BININT2 (0x4d) Push a two-byte unsigned integer. This is a space optimization for pickling small positive ints, in range(256, 2**16). Integers in range(256) can also be pickled via BININT2, but BININT1 instead saves a byte. ** Stack before: [] Stack after: [int] **/ func (pm *PickleMachine) opcode_BININT2() error { var v uint16 err := pm.readBinaryInto(&v, false) if err != nil { return err } pm.push(int64(v)) return nil } /** Opcode: BINSTRING (0x54) Push a Python string object. There are two arguments: the first is a 4-byte little-endian signed int giving the number of bytes in the string, and the second is that many bytes, which are taken literally as the string content. ** Stack before: [] Stack after: [str] **/ func (pm *PickleMachine) opcode_BINSTRING() error { var strlen int32 err := pm.readBinaryInto(&strlen, false) if err != nil { return err } if strlen < 0 { return fmt.Errorf("BINSTRING specified negative string length of %d", strlen) } str, err := pm.readFixedLengthString(int64(strlen)) if err != nil { return err } pm.push(str) return nil } /** Opcode: SHORT_BINSTRING (0x55) Push a Python string object. There are two arguments: the first is a 1-byte unsigned int giving the number of bytes in the string, and the second is that many bytes, which are taken literally as the string content. ** Stack before: [] Stack after: [str] **/ func (pm *PickleMachine) opcode_SHORT_BINSTRING() error { var strlen uint8 err := pm.readBinaryInto(&strlen, false) if err != nil { return err } if strlen < 0 { return fmt.Errorf("SHORT_BINSTRING specified negative string length of %d", strlen) } str, err := pm.readFixedLengthString(int64(strlen)) if err != nil { return err } pm.push(str) return nil } /** Opcode: BINUNICODE (0x58) Push a Python Unicode string object. There are two arguments: the first is a 4-byte little-endian signed int giving the number of bytes in the string. The second is that many bytes, and is the UTF-8 encoding of the Unicode string. ** Stack before: [] Stack after: [unicode] **/ func (pm *PickleMachine) opcode_BINUNICODE() error { var l int32 err := pm.readBinaryInto(&l, false) if err != nil { return err } str, err := pm.readFixedLengthString(int64(l)) if err != nil { return err } pm.push(str) return nil } /** Opcode: BINFLOAT (0x47) Float stored in binary form, with 8 bytes of data. This generally requires less than half the space of FLOAT encoding. In general, BINFLOAT cannot be used to transport infinities, NaNs, or minus zero, raises an exception if the exponent exceeds the range of an IEEE-754 double, and retains no more than 53 bits of precision (if there are more than that, "add a half and chop" rounding is used to cut it back to 53 significant bits). ** Stack before: [] Stack after: [float] **/ func (pm *PickleMachine) opcode_BINFLOAT() error { var v float64 err := pm.readBinaryInto(&v, true) if err != nil { return err } pm.push(v) return nil } /** Opcode: EMPTY_LIST (0x5d) Push an empty list.** Stack before: [] Stack after: [list] **/ func (pm *PickleMachine) opcode_EMPTY_LIST() error { v := make([]interface{}, 0) pm.push(v) return nil } /** Opcode: APPENDS (0x65) Extend a list by a slice of stack objects. Stack before: ... pylist markobject stackslice Stack after: ... pylist+stackslice although pylist is really extended in-place. ** Stack before: [list, mark, stackslice] Stack after: [list] **/ func (pm *PickleMachine) opcode_APPENDS() error { markIndex, err := pm.findMark() if err != nil { return err } pyListI, err := pm.readFromStackAt(markIndex - 1) if err != nil { return err } pyList, ok := pyListI.([]interface{}) if !ok { return fmt.Errorf("APPENDS expected type %T but got (%v)%T", pyList, pyListI, pyListI) } pyList = append(pyList, pm.Stack[markIndex+1:]...) pm.popAfterIndex(markIndex - 1) /** if err != nil { return err }**/ pm.push(pyList) return nil } /** Opcode: EMPTY_TUPLE (0x29) Push an empty tuple.** Stack before: [] Stack after: [tuple] **/ func (pm *PickleMachine) opcode_EMPTY_TUPLE() error { return pm.opcode_EMPTY_LIST() } /** Opcode: EMPTY_DICT (0x7d) Push an empty dict.** Stack before: [] Stack after: [dict] **/ func (pm *PickleMachine) opcode_EMPTY_DICT() error { pm.push(make(map[interface{}]interface{})) return nil } /** Opcode: SETITEMS (0x75) Add an arbitrary number of key+value pairs to an existing dict. The slice of the stack following the topmost markobject is taken as an alternating sequence of keys and values, added to the dict immediately under the topmost markobject. Everything at and after the topmost markobject is popped, leaving the mutated dict at the top of the stack. Stack before: ... pydict markobject key_1 value_1 ... key_n value_n Stack after: ... pydict where pydict has been modified via pydict[key_i] = value_i for i in 1, 2, ..., n, and in that order. ** Stack before: [dict, mark, stackslice] Stack after: [dict] **/ func (pm *PickleMachine) opcode_SETITEMS() (err error) { defer func() { if r := recover(); r != nil { switch x := r.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } } }() markIndex, err := pm.findMark() if err != nil { return err } vI, err := pm.readFromStackAt(markIndex - 1) if err != nil { return err } v, ok := vI.(map[interface{}]interface{}) if !ok { return fmt.Errorf("Opcode SETITEMS expected type %T on stack but found %v(%T)", v, vI, vI) } if ((len(pm.Stack) - markIndex + 1) % 2) != 0 { return fmt.Errorf("Found odd number of items on stack after mark:%d", len(pm.Stack)-markIndex+1) } for i := markIndex + 1; i != len(pm.Stack); i++ { key := pm.Stack[i] i++ v[key] = pm.Stack[i] } pm.popAfterIndex(markIndex) return nil } /** Opcode: POP_MARK (0x31) Pop all the stack objects at and above the topmost markobject. When an opcode using a variable number of stack objects is done, POP_MARK is used to remove those objects, and to remove the markobject that delimited their starting position on the stack. ** Stack before: [mark, stackslice] Stack after: [] **/ func (pm *PickleMachine) opcode_POP_MARK() error { markIndex, err := pm.findMark() if err != nil { return nil } pm.popAfterIndex(markIndex) return nil } /** Opcode: BINGET (0x68) Read an object from the memo and push it on the stack. The index of the memo object to push is given by the 1-byte unsigned integer following. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_BINGET() (err error) { defer func() { if r := recover(); r != nil { switch x := r.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } } }() var index uint8 err = pm.readBinaryInto(&index, false) if err != nil { return err } v, err := pm.readFromMemo(int64(index)) if err != nil { return err } //TODO test if the object we are about to push is mutable //if so it needs to be somehow deep copied first pm.push(v) return nil } /** Opcode: LONG_BINGET (0x6a) Read an object from the memo and push it on the stack. The index of the memo object to push is given by the 4-byte signed little-endian integer following. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_LONG_BINGET() (err error) { defer func() { if r := recover(); r != nil { switch x := r.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } } }() var index int32 err = pm.readBinaryInto(&index, false) if err != nil { return err } v, err := pm.readFromMemo(int64(index)) if err != nil { return err } //TODO test if the object we are about to push is mutable //if so it needs to be somehow deep copied first pm.push(v) return nil } /** Opcode: BINPUT (0x71) Store the stack top into the memo. The stack is not popped. The index of the memo location to write into is given by the 1-byte unsigned integer following. ** Stack before: [] Stack after: [] **/ func (pm *PickleMachine) opcode_BINPUT() error { v, err := pm.readFromStack(0) if err != nil { return err } var index uint8 err = pm.readBinaryInto(&index, false) if err != nil { return err } pm.storeMemo(int64(index), v) return nil } /** Opcode: LONG_BINPUT (0x72) Store the stack top into the memo. The stack is not popped. The index of the memo location to write into is given by the 4-byte signed little-endian integer following. ** Stack before: [] Stack after: [] **/ func (pm *PickleMachine) opcode_LONG_BINPUT() error { var index int32 err := pm.readBinaryInto(&index, false) if err != nil { return err } v, err := pm.readFromStack(0) if err != nil { return err } err = pm.storeMemo(int64(index), v) if err != nil { return err } return nil } /** Opcode: OBJ (0x6f) Build a class instance. This is the protocol 1 version of protocol 0's INST opcode, and is very much like it. The major difference is that the class object is taken off the stack, allowing it to be retrieved from the memo repeatedly if several instances of the same class are created. This can be much more efficient (in both time and space) than repeatedly embedding the module and class names in INST opcodes. Unlike INST, OBJ takes no arguments from the opcode stream. Instead the class object is taken off the stack, immediately above the topmost markobject: Stack before: ... markobject classobject stackslice Stack after: ... new_instance_object As for INST, the remainder of the stack above the markobject is gathered into an argument tuple, and then the logic seems identical, except that no __safe_for_unpickling__ check is done (XXX this is a bug; cPickle does test __safe_for_unpickling__). See INST for the gory details. NOTE: In Python 2.3, INST and OBJ are identical except for how they get the class object. That was always the intent; the implementations had diverged for accidental reasons. ** Stack before: [mark, any, stackslice] Stack after: [any] **/ func (pm *PickleMachine) opcode_OBJ() error { return ErrOpcodeNotImplemented } /** Opcode: BINPERSID (0x51) Push an object identified by a persistent ID. Like PERSID, except the persistent ID is popped off the stack (instead of being a string embedded in the opcode bytestream). The persistent ID is passed to self.persistent_load(), and whatever object that returns is pushed on the stack. See PERSID for more detail. ** Stack before: [any] Stack after: [any] **/ func (pm *PickleMachine) opcode_BINPERSID() error { return ErrOpcodeNotImplemented } stalecucumber-master/helpers_test.go0000644000000000000000000000274413011632561016740 0ustar rootrootpackage stalecucumber import "reflect" import "testing" import "strings" import "math/big" func TestHelperDictString(t *testing.T) { result, err := DictString(Unpickle(strings.NewReader("\x80\x02}q\x00(U\x01aq\x01K\x01K\x02K\x03u."))) if err == nil { t.Fatalf("Should not have unpickled:%v", result) } reader := strings.NewReader("\x80\x02}q\x00(U\x01aq\x01K*U\x01cq\x02U\x06foobarq\x03U\x01bq\x04G@*\xbdp\xa3\xd7\n=U\x01eq\x05\x88U\x01dq\x06\x8a\x01\x01u.") result, err = DictString(Unpickle(reader)) if err != nil { t.Fatal(err) } expect := make(map[string]interface{}) expect["a"] = int64(42) expect["b"] = 13.37 expect["c"] = "foobar" expect["d"] = big.NewInt(1) expect["e"] = true if !reflect.DeepEqual(expect, result) { t.Fatalf("Got %v expected %v", expect, result) } } func TestIntHelper(t *testing.T) { result, err := Int(Unpickle(strings.NewReader("\x80\x02\x8a\x08\xff\xff\xff\xff\xff\xff\xff\x7f."))) if err != nil { t.Fatal(err) } var expect int64 expect = 9223372036854775807 if result != expect { t.Fatalf("got %d expected %d", result, expect) } expect *= -1 result, err = Int(Unpickle(strings.NewReader("\x80\x02\x8a\x08\x01\x00\x00\x00\x00\x00\x00\x80."))) if err != nil { t.Fatal(err) } if result != expect { t.Fatalf("got %d expected %d", result, expect) } result, err = Int(Unpickle(strings.NewReader("\x80\x02\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\x01."))) if err == nil { t.Fatalf("should not have unpickled:%d", result) } } stalecucumber-master/opcodes.go0000644000000000000000000000267213011632561015673 0ustar rootrootpackage stalecucumber const OPCODE_INT = 0x49 const OPCODE_LONG = 0x4c const OPCODE_STRING = 0x53 const OPCODE_NONE = 0x4e const OPCODE_UNICODE = 0x56 const OPCODE_FLOAT = 0x46 const OPCODE_APPEND = 0x61 const OPCODE_LIST = 0x6c const OPCODE_TUPLE = 0x74 const OPCODE_DICT = 0x64 const OPCODE_SETITEM = 0x73 const OPCODE_POP = 0x30 const OPCODE_DUP = 0x32 const OPCODE_MARK = 0x28 const OPCODE_GET = 0x67 const OPCODE_PUT = 0x70 const OPCODE_GLOBAL = 0x63 const OPCODE_REDUCE = 0x52 const OPCODE_BUILD = 0x62 const OPCODE_INST = 0x69 const OPCODE_STOP = 0x2e const OPCODE_PERSID = 0x50 const OPCODE_BININT = 0x4a const OPCODE_BININT1 = 0x4b const OPCODE_BININT2 = 0x4d const OPCODE_BINSTRING = 0x54 const OPCODE_SHORT_BINSTRING = 0x55 const OPCODE_BINUNICODE = 0x58 const OPCODE_BINFLOAT = 0x47 const OPCODE_EMPTY_LIST = 0x5d const OPCODE_APPENDS = 0x65 const OPCODE_EMPTY_TUPLE = 0x29 const OPCODE_EMPTY_DICT = 0x7d const OPCODE_SETITEMS = 0x75 const OPCODE_POP_MARK = 0x31 const OPCODE_BINGET = 0x68 const OPCODE_LONG_BINGET = 0x6a const OPCODE_BINPUT = 0x71 const OPCODE_LONG_BINPUT = 0x72 const OPCODE_OBJ = 0x6f const OPCODE_BINPERSID = 0x51 const OPCODE_LONG1 = 0x8a const OPCODE_LONG4 = 0x8b const OPCODE_NEWTRUE = 0x88 const OPCODE_NEWFALSE = 0x89 const OPCODE_TUPLE1 = 0x85 const OPCODE_TUPLE2 = 0x86 const OPCODE_TUPLE3 = 0x87 const OPCODE_EXT1 = 0x82 const OPCODE_EXT2 = 0x83 const OPCODE_EXT4 = 0x84 const OPCODE_NEWOBJ = 0x81 const OPCODE_PROTO = 0x80 stalecucumber-master/README.md0000644000000000000000000000273313011632561015165 0ustar rootroot#stalecucumber This package reads and writes pickled data. The format is the same as the Python "pickle" module. Protocols 0,1,2 are implemented. These are the versions written by the Python 2.x series. Python 3 defines newer protocol versions, but can write the older protocol versions so they are readable by this package. [Full documentation is available here.](https://godoc.org/github.com/hydrogen18/stalecucumber) ##TLDR Read a pickled string or unicode object ``` pickle.dumps("foobar") --- var somePickledData io.Reader mystring, err := stalecucumber.String(stalecucumber.Unpickle(somePickledData)) ```` Read a pickled integer ``` pickle.dumps(42) --- var somePickledData io.Reader myint64, err := stalecucumber.Int(stalecucumber.Unpickle(somePickledData)) ``` Read a pickled list of numbers into a structure ``` pickle.dumps([8,8,2005]) --- var somePickledData io.Reader numbers := make([]int64,0) err := stalecucumber.UnpackInto(&numbers).From(stalecucumber.Unpickle(somePickledData)) ``` Read a pickled dictionary into a structure ``` pickle.dumps({"apple":1,"banana":2,"cat":"hello","Dog":42.0}) --- var somePickledData io.Reader mystruct := struct{ Apple int Banana uint Cat string Dog float32}{} err := stalecucumber.UnpackInto(&mystruct).From(stalecucumber.Unpickle(somePickledData)) ``` Pickle a struct ``` buf := new(bytes.Buffer) mystruct := struct{ Apple int Banana uint Cat string Dog float32}{} err := stalecucumber.NewPickler(buf).Pickle(mystruct) ``` stalecucumber-master/pickle_machine.go0000644000000000000000000003630413011632561017171 0ustar rootroot/* This package reads and writes pickled data. The format is the same as the Python "pickle" module. Protocols 0,1,2 are implemented. These are the versions written by the Python 2.x series. Python 3 defines newer protocol versions, but can write the older protocol versions so they are readable by this package. To read data, see stalecucumber.Unpickle. To write data, see stalecucumber.NewPickler. TLDR Read a pickled string or unicode object pickle.dumps("foobar") --- var somePickledData io.Reader mystring, err := stalecucumber.String(stalecucumber.Unpickle(somePickledData)) Read a pickled integer pickle.dumps(42) --- var somePickledData io.Reader myint64, err := stalecucumber.Int(stalecucumber.Unpickle(somePickledData)) Read a pickled list of numbers into a structure pickle.dumps([8,8,2005]) --- var somePickledData io.Reader numbers := make([]int64,0) err := stalecucumber.UnpackInto(&numbers).From(stalecucumber.Unpickle(somePickledData)) Read a pickled dictionary into a structure pickle.dumps({"apple":1,"banana":2,"cat":"hello","Dog":42.0}) --- var somePickledData io.Reader mystruct := struct{ Apple int Banana uint Cat string Dog float32}{} err := stalecucumber.UnpackInto(&mystruct).From(stalecucumber.Unpickle(somePickledData)) Pickle a struct buf := new(bytes.Buffer) mystruct := struct{ Apple int Banana uint Cat string Dog float32}{} err := stalecucumber.NewPickler(buf).Pickle(mystruct) Recursive objects You can pickle recursive objects like so a = {} a["self"] = a pickle.dumps(a) Python's pickler is intelligent enough not to emit an infinite data structure when a recursive object is pickled. I recommend against pickling recursive objects in the first place, but this library handles unpickling them without a problem. The result of unpickling the above is map[interface{}]interface{} with a key "a" that contains a reference to itself. Attempting to unpack the result of the above python code into a structure with UnpackInto would either fail or recurse forever. Protocol Performance If the version of Python you are using supports protocol version 1 or 2, you should always specify that protocol version. By default the "pickle" and "cPickle" modules in Python write using protocol 0. Protocol 0 requires much more space to represent the same values and is much slower to parse. Unsupported Opcodes The pickle format is incredibly flexible and as a result has some features that are impractical or unimportant when implementing a reader in another language. Each set of opcodes is listed below by protocol version with the impact. Protocol 0 GLOBAL This opcode is equivalent to calling "import foo; foo.bar" in python. It is generated whenever an object instance, class definition, or method definition is serialized. As long as the pickled data does not contain an instance of a python class or a reference to a python callable this opcode is not emitted by the "pickle" module. A few examples of what will definitely cause this opcode to be emitted pickle.dumps(range) #Pickling the range function pickle.dumps(Exception()) #Pickling an instance of a python class This opcode will be partially supported in a future revision to this package that allows the unpickling of instances of Python classes. REDUCE BUILD INST These opcodes are used in recreating pickled python objects. That is currently not supported by this package. These opcodes will be supported in a future revision to this package that allows the unpickling of instances of Python classes. PERSID This opcode is used to reference concrete definitions of objects between a pickler and an unpickler by an ID number. The pickle protocol doesn't define what a persistent ID means. This opcode is unlikely to ever be supported by this package. Protocol 1 OBJ This opcodes is used in recreating pickled python objects. That is currently not supported by this package. This opcode will supported in a future revision to this package that allows the unpickling of instances of Python classes. BINPERSID This opcode is equivalent to PERSID in protocol 0 and won't be supported for the same reason. Protocol 2 NEWOBJ This opcodes is used in recreating pickled python objects. That is currently not supported by this package. This opcode will supported in a future revision to this package that allows the unpickling of instances of Python classes. EXT1 EXT2 EXT4 These opcodes allow using a registry of popular objects that are pickled by name, typically classes. It is envisioned that through a global negotiation and registration process, third parties can set up a mapping between ints and object names. These opcodes are unlikely to ever be supported by this package. */ package stalecucumber import "errors" import "io" import "bytes" import "encoding/binary" import "fmt" var ErrOpcodeStopped = errors.New("STOP opcode found") var ErrStackTooSmall = errors.New("Stack is too small to perform requested operation") var ErrInputTruncated = errors.New("Input to the pickle machine was truncated") var ErrOpcodeNotImplemented = errors.New("Input encountered opcode that is not implemented") var ErrNoResult = errors.New("Input did not place a value onto the stack") var ErrMarkNotFound = errors.New("Mark could not be found on the stack") /* Unpickle a value from a reader. This function takes a reader and attempts to read a complete pickle program from it. This is normally the output of the function "pickle.dump" from Python. The returned type is interface{} because unpickling can generate any type. Use a helper function to convert to another type without an additional type check. This function returns an error if the reader fails, the pickled data is invalid, or if the pickled data contains an unsupported opcode. See unsupported opcodes in the documentation of this package for more information. Type Conversions Types conversion Python types to Go types is performed as followed int -> int64 string -> string unicode -> string float -> float64 long -> big.Int from the "math/big" package lists -> []interface{} tuples -> []interface{} dict -> map[interface{}]interface{} The following values are converted from Python to the Go types True & False -> bool None -> stalecucumber.PickleNone, sets pointers to nil Helper Functions The following helper functions were inspired by the github.com/garyburd/redigo package. Each function takes the result of Unpickle as its arguments. If unpickle fails it does nothing and returns that error. Otherwise it attempts to convert to the appropriate type. If type conversion fails it returns an error String - string from Python string or unicode Int - int64 from Python int or long Bool - bool from Python True or False Big - *big.Int from Python long ListOrTuple - []interface{} from Python Tuple or List Float - float64 from Python float Dict - map[interface{}]interface{} from Python dictionary DictString - map[string]interface{} from Python dictionary. Keys must all be of type unicode or string. Unpacking into structures If the pickled object is a python dictionary that has only unicode and string objects for keys, that object can be unpickled into a struct in Go by using the "UnpackInto" function. The "From" receiver on the return value accepts the result of "Unpickle" as its actual parameters. The keys of the python dictionary are assigned to fields in a structure. Structures may specify the tag "pickle" on fields. The value of this tag is taken as the key name of the Python dictionary value to place in this field. If no field has a matching "pickle" tag the fields are looked up by name. If the first character of the key is not uppercase, it is uppercased. If a field matching that name is found, the value in the python dictionary is unpacked into the value of the field within the structure. A list of python dictionaries can be unpickled into a slice of structures in Go. A homogeneous list of python values can be unpickled into a slice in Go with the appropriate element type. A nested python dictionary is unpickled into nested structures in Go. If a field is of type map[interface{}]interface{} it is of course unpacked into that as well. By default UnpackInto skips any missing fields and fails if a field's type is not compatible with the object's type. This behavior can be changed by setting "AllowMissingFields" and "AllowMismatchedFields" on the return value of UnpackInto before calling From. */ func Unpickle(reader io.Reader) (interface{}, error) { var pm PickleMachine pm.buf = &bytes.Buffer{} pm.Reader = reader pm.lastMark = -1 //Pre allocate a small stack pm.Stack = make([]interface{}, 0, 16) err := (&pm).execute() if err != ErrOpcodeStopped { return nil, pm.error(err) } if len(pm.Stack) == 0 { return nil, ErrNoResult } return pm.Stack[0], nil } var jumpList = buildEmptyJumpList() func init() { populateJumpList(&jumpList) } /* This type is returned whenever Unpickle encounters an error in pickled data. */ type PickleMachineError struct { Err error StackSize int MemoSize int Opcode uint8 } /* This struct is current exposed but not useful. It is likely to be hidden in the near future. */ type PickleMachine struct { Stack []interface{} Memo []interface{} Reader io.Reader currentOpcode uint8 buf *bytes.Buffer lastMark int memoBuffer [16]memoBufferElement memoBufferMaxDestination int64 memoBufferIndex int } type memoBufferElement struct { Destination int64 V interface{} } func (pme PickleMachineError) Error() string { return fmt.Sprintf("Pickle Machine failed on opcode:0x%x. Stack size:%d. Memo size:%d. Cause:%v", pme.Opcode, pme.StackSize, pme.MemoSize, pme.Err) } func (pm *PickleMachine) error(src error) error { return PickleMachineError{ StackSize: len(pm.Stack), MemoSize: len(pm.Memo), Err: src, Opcode: pm.currentOpcode, } } func (pm *PickleMachine) execute() error { for { err := binary.Read(pm.Reader, binary.BigEndian, &pm.currentOpcode) if err != nil { return err } err = jumpList[int(pm.currentOpcode)](pm) if err != nil { return err } } } func (pm *PickleMachine) flushMemoBuffer(vIndex int64, v interface{}) { //Extend the memo until it is large enough if pm.memoBufferMaxDestination >= int64(len(pm.Memo)) { replacement := make([]interface{}, pm.memoBufferMaxDestination<<1) copy(replacement, pm.Memo) pm.Memo = replacement } //If a value was passed into this function, write it into the memo //as well if vIndex != -1 { pm.Memo[vIndex] = v } //Write the contents of the buffer into the memo //in the same order as the puts were issued for i := 0; i != pm.memoBufferIndex; i++ { buffered := pm.memoBuffer[i] pm.Memo[buffered.Destination] = buffered.V } //Reset the buffer pm.memoBufferIndex = 0 pm.memoBufferMaxDestination = 0 return } func (pm *PickleMachine) storeMemo(index int64, v interface{}) error { if index < 0 { return fmt.Errorf("Requested to write to invalid memo index:%v", index) } //If there is space in the memo presently, then store it //and it is done. if index < int64(len(pm.Memo)) { pm.Memo[index] = v return nil } //Update the maximum index in the buffer if need be if index > pm.memoBufferMaxDestination { pm.memoBufferMaxDestination = index } //If the buffer is not full write into it if pm.memoBufferIndex != len(pm.memoBuffer) { pm.memoBuffer[pm.memoBufferIndex].V = v pm.memoBuffer[pm.memoBufferIndex].Destination = index pm.memoBufferIndex++ } else { //If the buffer is full flush it now pm.flushMemoBuffer(index, v) } return nil } func (pm *PickleMachine) readFromMemo(index int64) (interface{}, error) { if index < 0 { return nil, fmt.Errorf("Requested to read from negative memo index %d", index) } //Test to see if the value is outside the current length of the memo if index >= int64(len(pm.Memo)) { pm.flushMemoBuffer(-1, nil) if index >= int64(len(pm.Memo)) { return nil, fmt.Errorf("Requested to read from invalid memo index %d", index) } } //Grab the value retval := pm.Memo[index] //If nil then flush the memo buffer to see if it is within it if retval == nil { pm.flushMemoBuffer(-1, nil) //Grab the value again after the flush retval = pm.Memo[index] //If still nil, then this is a read from an invalid position if retval == nil { return nil, fmt.Errorf("Requested to read from invalid memo index %d", index) } } return retval, nil } func (pm *PickleMachine) push(v interface{}) { pm.Stack = append(pm.Stack, v) } func (pm *PickleMachine) pop() (interface{}, error) { l := len(pm.Stack) if l == 0 { return nil, ErrStackTooSmall } l-- top := pm.Stack[l] pm.Stack = pm.Stack[:l] return top, nil } func (pm *PickleMachine) readFromStack(offset int) (interface{}, error) { return pm.readFromStackAt(len(pm.Stack) - 1 - offset) } func (pm *PickleMachine) readFromStackAt(position int) (interface{}, error) { if position < 0 { return nil, fmt.Errorf("Request to read from invalid stack position %d", position) } return pm.Stack[position], nil } func (pm *PickleMachine) readIntFromStack(offset int) (int64, error) { v, err := pm.readFromStack(offset) if err != nil { return 0, err } vi, ok := v.(int64) if !ok { return 0, fmt.Errorf("Type %T was requested from stack but found %v(%T)", vi, v, v) } return vi, nil } func (pm *PickleMachine) popAfterIndex(index int) { //Input to this function must be sane, no checking is done /** if len(pm.Stack)-1 < index { return ErrStackTooSmall }**/ pm.Stack = pm.Stack[0:index] } func (pm *PickleMachine) findMark() (int, error) { if pm.lastMark != -1 { mark := pm.lastMark pm.lastMark = -1 if mark < len(pm.Stack) { if _, ok := pm.Stack[mark].(PickleMark); ok { return mark, nil } } } for i := len(pm.Stack) - 1; i != -1; i-- { if _, ok := pm.Stack[i].(PickleMark); ok { return i, nil } } return -1, ErrMarkNotFound } func (pm *PickleMachine) readFixedLengthRaw(l int64) ([]byte, error) { pm.buf.Reset() _, err := io.CopyN(pm.buf, pm.Reader, l) if err != nil { return nil, err } return pm.buf.Bytes(), nil } func (pm *PickleMachine) readFixedLengthString(l int64) (string, error) { //Avoid getting "" if l == 0 { return "", nil } pm.buf.Reset() _, err := io.CopyN(pm.buf, pm.Reader, l) if err != nil { return "", err } return pm.buf.String(), nil } func (pm *PickleMachine) readBytes() ([]byte, error) { //This is slow and protocol 0 only pm.buf.Reset() for { var v [1]byte n, err := pm.Reader.Read(v[:]) if n != 1 { return nil, ErrInputTruncated } if err != nil { return nil, err } if v[0] == '\n' { break } pm.buf.WriteByte(v[0]) } return pm.buf.Bytes(), nil } func (pm *PickleMachine) readString() (string, error) { //This is slow and protocol 0 only pm.buf.Reset() for { var v [1]byte n, err := pm.Reader.Read(v[:]) if n != 1 { return "", ErrInputTruncated } if err != nil { return "", err } if v[0] == '\n' { break } pm.buf.WriteByte(v[0]) } //Avoid getting "" if pm.buf.Len() == 0 { return "", nil } return pm.buf.String(), nil } func (pm *PickleMachine) readBinaryInto(dst interface{}, bigEndian bool) error { var bo binary.ByteOrder if bigEndian { bo = binary.BigEndian } else { bo = binary.LittleEndian } return binary.Read(pm.Reader, bo, dst) } stalecucumber-master/populate_jump_list.go0000644000000000000000000000544213011632561020154 0ustar rootrootpackage stalecucumber func populateJumpList(jl *opcodeJumpList) { jl[OPCODE_INT] = (*PickleMachine).opcode_INT jl[OPCODE_LONG] = (*PickleMachine).opcode_LONG jl[OPCODE_STRING] = (*PickleMachine).opcode_STRING jl[OPCODE_NONE] = (*PickleMachine).opcode_NONE jl[OPCODE_UNICODE] = (*PickleMachine).opcode_UNICODE jl[OPCODE_FLOAT] = (*PickleMachine).opcode_FLOAT jl[OPCODE_APPEND] = (*PickleMachine).opcode_APPEND jl[OPCODE_LIST] = (*PickleMachine).opcode_LIST jl[OPCODE_TUPLE] = (*PickleMachine).opcode_TUPLE jl[OPCODE_DICT] = (*PickleMachine).opcode_DICT jl[OPCODE_SETITEM] = (*PickleMachine).opcode_SETITEM jl[OPCODE_POP] = (*PickleMachine).opcode_POP jl[OPCODE_DUP] = (*PickleMachine).opcode_DUP jl[OPCODE_MARK] = (*PickleMachine).opcode_MARK jl[OPCODE_GET] = (*PickleMachine).opcode_GET jl[OPCODE_PUT] = (*PickleMachine).opcode_PUT jl[OPCODE_GLOBAL] = (*PickleMachine).opcode_GLOBAL jl[OPCODE_REDUCE] = (*PickleMachine).opcode_REDUCE jl[OPCODE_BUILD] = (*PickleMachine).opcode_BUILD jl[OPCODE_INST] = (*PickleMachine).opcode_INST jl[OPCODE_STOP] = (*PickleMachine).opcode_STOP jl[OPCODE_PERSID] = (*PickleMachine).opcode_PERSID jl[OPCODE_BININT] = (*PickleMachine).opcode_BININT jl[OPCODE_BININT1] = (*PickleMachine).opcode_BININT1 jl[OPCODE_BININT2] = (*PickleMachine).opcode_BININT2 jl[OPCODE_BINSTRING] = (*PickleMachine).opcode_BINSTRING jl[OPCODE_SHORT_BINSTRING] = (*PickleMachine).opcode_SHORT_BINSTRING jl[OPCODE_BINUNICODE] = (*PickleMachine).opcode_BINUNICODE jl[OPCODE_BINFLOAT] = (*PickleMachine).opcode_BINFLOAT jl[OPCODE_EMPTY_LIST] = (*PickleMachine).opcode_EMPTY_LIST jl[OPCODE_APPENDS] = (*PickleMachine).opcode_APPENDS jl[OPCODE_EMPTY_TUPLE] = (*PickleMachine).opcode_EMPTY_TUPLE jl[OPCODE_EMPTY_DICT] = (*PickleMachine).opcode_EMPTY_DICT jl[OPCODE_SETITEMS] = (*PickleMachine).opcode_SETITEMS jl[OPCODE_POP_MARK] = (*PickleMachine).opcode_POP_MARK jl[OPCODE_BINGET] = (*PickleMachine).opcode_BINGET jl[OPCODE_LONG_BINGET] = (*PickleMachine).opcode_LONG_BINGET jl[OPCODE_BINPUT] = (*PickleMachine).opcode_BINPUT jl[OPCODE_LONG_BINPUT] = (*PickleMachine).opcode_LONG_BINPUT jl[OPCODE_OBJ] = (*PickleMachine).opcode_OBJ jl[OPCODE_BINPERSID] = (*PickleMachine).opcode_BINPERSID jl[OPCODE_LONG1] = (*PickleMachine).opcode_LONG1 jl[OPCODE_LONG4] = (*PickleMachine).opcode_LONG4 jl[OPCODE_NEWTRUE] = (*PickleMachine).opcode_NEWTRUE jl[OPCODE_NEWFALSE] = (*PickleMachine).opcode_NEWFALSE jl[OPCODE_TUPLE1] = (*PickleMachine).opcode_TUPLE1 jl[OPCODE_TUPLE2] = (*PickleMachine).opcode_TUPLE2 jl[OPCODE_TUPLE3] = (*PickleMachine).opcode_TUPLE3 jl[OPCODE_EXT1] = (*PickleMachine).opcode_EXT1 jl[OPCODE_EXT2] = (*PickleMachine).opcode_EXT2 jl[OPCODE_EXT4] = (*PickleMachine).opcode_EXT4 jl[OPCODE_NEWOBJ] = (*PickleMachine).opcode_NEWOBJ jl[OPCODE_PROTO] = (*PickleMachine).opcode_PROTO } stalecucumber-master/protocol_0.go0000644000000000000000000004433413011632561016320 0ustar rootrootpackage stalecucumber import "strconv" import "fmt" import "math/big" import "errors" //import "unicode/utf8" import "unicode/utf16" /** Opcode: INT Push an integer or bool. The argument is a newline-terminated decimal literal string. The intent may have been that this always fit in a short Python int, but INT can be generated in pickles written on a 64-bit box that require a Python long on a 32-bit box. The difference between this and LONG then is that INT skips a trailing 'L', and produces a short int whenever possible. Another difference is due to that, when bool was introduced as a distinct type in 2.3, builtin names True and False were also added to 2.2.2, mapping to ints 1 and 0. For compatibility in both directions, True gets pickled as INT + "I01\n", and False as INT + "I00\n". Leading zeroes are never produced for a genuine integer. The 2.3 (and later) unpicklers special-case these and return bool instead; earlier unpicklers ignore the leading "0" and return the int. ** Stack before: [] Stack after: [int_or_bool] **/ func (pm *PickleMachine) opcode_INT() error { str, err := pm.readString() if err != nil { return err } //check for boolean sentinels if len(str) == 2 { switch str { case "01": pm.push(true) return nil case "00": pm.push(false) return nil default: } } n, err := strconv.ParseInt(str, 10, 64) if err != nil { return err } pm.push(n) return nil } /** Opcode: LONG Push a long integer. The same as INT, except that the literal ends with 'L', and always unpickles to a Python long. There doesn't seem a real purpose to the trailing 'L'. Note that LONG takes time quadratic in the number of digits when unpickling (this is simply due to the nature of decimal->binary conversion). Proto 2 added linear-time (in C; still quadratic-time in Python) LONG1 and LONG4 opcodes. ** Stack before: [] Stack after: [long] **/ func (pm *PickleMachine) opcode_LONG() error { i := new(big.Int) str, err := pm.readString() if err != nil { return err } if len(str) == 0 { return fmt.Errorf("String for LONG opcode cannot be zero length") } last := str[len(str)-1] if last != 'L' { return fmt.Errorf("String for LONG opcode must end with %q not %q", "L", last) } v := str[:len(str)-1] _, err = fmt.Sscan(v, i) if err != nil { return err } pm.push(i) return nil } /** Opcode: STRING Push a Python string object. The argument is a repr-style string, with bracketing quote characters, and perhaps embedded escapes. The argument extends until the next newline character. ** Stack before: [] Stack after: [str] **/ var unquoteInputs = []byte{0x27, 0x22, 0x0} func (pm *PickleMachine) opcode_STRING() error { str, err := pm.readString() if err != nil { return err } //For whatever reason, the string is quoted. So the first and last character //should always be the single quote if len(str) < 2 { return fmt.Errorf("For STRING opcode, argument has invalid length %d", len(str)) } if str[0] != '\'' || str[len(str)-1] != '\'' { return fmt.Errorf("For STRING opcode, argument has poorly formed value %q", str) } v := str[1 : len(str)-1] f := make([]rune, 0, len(v)) for len(v) != 0 { var vr rune var replacement string for _, i := range unquoteInputs { vr, _, replacement, err = strconv.UnquoteChar(v, i) if err == nil { break } } if err != nil { c := v[0] return fmt.Errorf("Read thus far %q. Failed to unquote character %c error:%v", string(f), c, err) } v = replacement f = append(f, vr) } pm.push(string(f)) return nil } /** Opcode: NONE Push None on the stack.** Stack before: [] Stack after: [None] **/ func (pm *PickleMachine) opcode_NONE() error { pm.push(PickleNone{}) return nil } /** Opcode: UNICODE Push a Python Unicode string object. The argument is a raw-unicode-escape encoding of a Unicode string, and so may contain embedded escape sequences. The argument extends until the next newline character. ** Stack before: [] Stack after: [unicode] **/ func (pm *PickleMachine) opcode_UNICODE() error { str, err := pm.readBytes() if err != nil { return err } f := make([]rune, 0, len(str)) var total int var consumed int total = len(str) for total != consumed { h := str[consumed] //Python 'raw-unicode-escape' doesnt //escape extended ascii if h > 127 { ea := utf16.Decode([]uint16{uint16(h)}) f = append(f, ea...) consumed += 1 continue } //Multibyte unicode points are escaped //so use "UnquoteChar" to handle those var vr rune for _, i := range unquoteInputs { pre := string(str[consumed:]) var post string vr, _, post, err = strconv.UnquoteChar(pre, i) if err == nil { consumed += len(pre) - len(post) break } } if err != nil { c := str[0] return fmt.Errorf("Read thus far %q. Failed to unquote character %c error:%v", string(f), c, err) } f = append(f, vr) } pm.push(string(f)) return nil } /** Opcode: FLOAT Newline-terminated decimal float literal. The argument is repr(a_float), and in general requires 17 significant digits for roundtrip conversion to be an identity (this is so for IEEE-754 double precision values, which is what Python float maps to on most boxes). In general, FLOAT cannot be used to transport infinities, NaNs, or minus zero across boxes (or even on a single box, if the platform C library can't read the strings it produces for such things -- Windows is like that), but may do less damage than BINFLOAT on boxes with greater precision or dynamic range than IEEE-754 double. ** Stack before: [] Stack after: [float] **/ func (pm *PickleMachine) opcode_FLOAT() error { str, err := pm.readString() if err != nil { return err } var v float64 _, err = fmt.Sscanf(str, "%f", &v) if err != nil { return err } pm.push(v) return nil } /** Opcode: APPEND Append an object to a list. Stack before: ... pylist anyobject Stack after: ... pylist+[anyobject] although pylist is really extended in-place. ** Stack before: [list, any] Stack after: [list] **/ func (pm *PickleMachine) opcode_APPEND() error { v, err := pm.pop() if err != nil { return err } listI, err := pm.pop() if err != nil { return err } list, ok := listI.([]interface{}) if !ok { fmt.Errorf("Second item on top of stack must be of %T not %T", list, listI) } list = append(list, v) pm.push(list) return nil } /** Opcode: LIST Build a list out of the topmost stack slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python list, which single list object replaces all of the stack from the topmost markobject onward. For example, Stack before: ... markobject 1 2 3 'abc' Stack after: ... [1, 2, 3, 'abc'] ** Stack before: [mark, stackslice] Stack after: [list] **/ func (pm *PickleMachine) opcode_LIST() error { markIndex, err := pm.findMark() if err != nil { return err } v := make([]interface{}, 0) for i := markIndex + 1; i != len(pm.Stack); i++ { v = append(v, pm.Stack[i]) } //Pop the values off the stack pm.popAfterIndex(markIndex) pm.push(v) return nil } /** Opcode: TUPLE Build a tuple out of the topmost stack slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python tuple, which single tuple object replaces all of the stack from the topmost markobject onward. For example, Stack before: ... markobject 1 2 3 'abc' Stack after: ... (1, 2, 3, 'abc') ** Stack before: [mark, stackslice] Stack after: [tuple] **/ func (pm *PickleMachine) opcode_TUPLE() error { return pm.opcode_LIST() } /** Opcode: DICT Build a dict out of the topmost stack slice, after markobject. All the stack entries following the topmost markobject are placed into a single Python dict, which single dict object replaces all of the stack from the topmost markobject onward. The stack slice alternates key, value, key, value, .... For example, Stack before: ... markobject 1 2 3 'abc' Stack after: ... {1: 2, 3: 'abc'} ** Stack before: [mark, stackslice] Stack after: [dict] **/ func (pm *PickleMachine) opcode_DICT() (err error) { defer func() { if r := recover(); r != nil { switch x := r.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } } }() markIndex, err := pm.findMark() if err != nil { return err } v := make(map[interface{}]interface{}) var key interface{} for i := markIndex + 1; i != len(pm.Stack); i++ { if key == nil { key = pm.Stack[i] } else { v[key] = pm.Stack[i] key = nil } } if key != nil { return fmt.Errorf("For opcode DICT stack after mark contained an odd number of items, this is not valid") } pm.popAfterIndex(markIndex) pm.push(v) return nil } /** Opcode: SETITEM Add a key+value pair to an existing dict. Stack before: ... pydict key value Stack after: ... pydict where pydict has been modified via pydict[key] = value. ** Stack before: [dict, any, any] Stack after: [dict] **/ func (pm *PickleMachine) opcode_SETITEM() (err error) { defer func() { if r := recover(); r != nil { switch x := r.(type) { case string: err = errors.New(x) case error: err = x default: err = errors.New("Unknown panic") } } }() v, err := pm.pop() if err != nil { return err } k, err := pm.pop() if err != nil { return err } dictI, err := pm.pop() if err != nil { return err } dict, ok := dictI.(map[interface{}]interface{}) if !ok { return fmt.Errorf("For opcode SETITEM stack item 2 from top must be of type %T not %T", dict, dictI) } dict[k] = v pm.push(dict) return nil } /** Opcode: POP Discard the top stack item, shrinking the stack by one item.** Stack before: [any] Stack after: [] **/ func (pm *PickleMachine) opcode_POP() error { _, err := pm.pop() return err } /** Opcode: DUP Push the top stack item onto the stack again, duplicating it.** Stack before: [any] Stack after: [any, any] **/ func (pm *PickleMachine) opcode_DUP() error { return ErrOpcodeNotImplemented } /** Opcode: MARK Push markobject onto the stack. markobject is a unique object, used by other opcodes to identify a region of the stack containing a variable number of objects for them to work on. See markobject.doc for more detail. ** Stack before: [] Stack after: [mark] **/ func (pm *PickleMachine) opcode_MARK() error { pm.lastMark = len(pm.Stack) pm.push(PickleMark{}) return nil } /** Opcode: GET Read an object from the memo and push it on the stack. The index of the memo object to push is given by the newline-terminated decimal string following. BINGET and LONG_BINGET are space-optimized versions. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_GET() error { str, err := pm.readString() if err != nil { return err } index, err := strconv.Atoi(str) if err != nil { return err } v, err := pm.readFromMemo(int64(index)) if err != nil { return err } //TODO test if the object we are about to push is mutable //if so it needs to be somehow deep copied first pm.push(v) return nil } /** Opcode: PUT Store the stack top into the memo. The stack is not popped. The index of the memo location to write into is given by the newline- terminated decimal string following. BINPUT and LONG_BINPUT are space-optimized versions. ** Stack before: [] Stack after: [] **/ func (pm *PickleMachine) opcode_PUT() error { if len(pm.Stack) < 1 { return ErrStackTooSmall } str, err := pm.readString() if err != nil { return err } idx, err := strconv.Atoi(str) if err != nil { return err } pm.storeMemo(int64(idx), pm.Stack[len(pm.Stack)-1]) return nil } /** Opcode: GLOBAL Push a global object (module.attr) on the stack. Two newline-terminated strings follow the GLOBAL opcode. The first is taken as a module name, and the second as a class name. The class object module.class is pushed on the stack. More accurately, the object returned by self.find_class(module, class) is pushed on the stack, so unpickling subclasses can override this form of lookup. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_GLOBAL() error { //TODO push an object that represents the result of this operation return ErrOpcodeNotImplemented } /** Opcode: REDUCE Push an object built from a callable and an argument tuple. The opcode is named to remind of the __reduce__() method. Stack before: ... callable pytuple Stack after: ... callable(*pytuple) The callable and the argument tuple are the first two items returned by a __reduce__ method. Applying the callable to the argtuple is supposed to reproduce the original object, or at least get it started. If the __reduce__ method returns a 3-tuple, the last component is an argument to be passed to the object's __setstate__, and then the REDUCE opcode is followed by code to create setstate's argument, and then a BUILD opcode to apply __setstate__ to that argument. If type(callable) is not ClassType, REDUCE complains unless the callable has been registered with the copy_reg module's safe_constructors dict, or the callable has a magic '__safe_for_unpickling__' attribute with a true value. I'm not sure why it does this, but I've sure seen this complaint often enough when I didn't want to . ** Stack before: [any, any] Stack after: [any] **/ func (pm *PickleMachine) opcode_REDUCE() error { //TODO push an object that represents the result result of this operation return ErrOpcodeNotImplemented } /** Opcode: BUILD Finish building an object, via __setstate__ or dict update. Stack before: ... anyobject argument Stack after: ... anyobject where anyobject may have been mutated, as follows: If the object has a __setstate__ method, anyobject.__setstate__(argument) is called. Else the argument must be a dict, the object must have a __dict__, and the object is updated via anyobject.__dict__.update(argument) This may raise RuntimeError in restricted execution mode (which disallows access to __dict__ directly); in that case, the object is updated instead via for k, v in argument.items(): anyobject[k] = v ** Stack before: [any, any] Stack after: [any] **/ func (pm *PickleMachine) opcode_BUILD() error { return ErrOpcodeNotImplemented } /** Opcode: INST Build a class instance. This is the protocol 0 version of protocol 1's OBJ opcode. INST is followed by two newline-terminated strings, giving a module and class name, just as for the GLOBAL opcode (and see GLOBAL for more details about that). self.find_class(module, name) is used to get a class object. In addition, all the objects on the stack following the topmost markobject are gathered into a tuple and popped (along with the topmost markobject), just as for the TUPLE opcode. Now it gets complicated. If all of these are true: + The argtuple is empty (markobject was at the top of the stack at the start). + It's an old-style class object (the type of the class object is ClassType). + The class object does not have a __getinitargs__ attribute. then we want to create an old-style class instance without invoking its __init__() method (pickle has waffled on this over the years; not calling __init__() is current wisdom). In this case, an instance of an old-style dummy class is created, and then we try to rebind its __class__ attribute to the desired class object. If this succeeds, the new instance object is pushed on the stack, and we're done. In restricted execution mode it can fail (assignment to __class__ is disallowed), and I'm not really sure what happens then -- it looks like the code ends up calling the class object's __init__ anyway, via falling into the next case. Else (the argtuple is not empty, it's not an old-style class object, or the class object does have a __getinitargs__ attribute), the code first insists that the class object have a __safe_for_unpickling__ attribute. Unlike as for the __safe_for_unpickling__ check in REDUCE, it doesn't matter whether this attribute has a true or false value, it only matters whether it exists (XXX this is a bug; cPickle requires the attribute to be true). If __safe_for_unpickling__ doesn't exist, UnpicklingError is raised. Else (the class object does have a __safe_for_unpickling__ attr), the class object obtained from INST's arguments is applied to the argtuple obtained from the stack, and the resulting instance object is pushed on the stack. NOTE: checks for __safe_for_unpickling__ went away in Python 2.3. ** Stack before: [mark, stackslice] Stack after: [any] **/ func (pm *PickleMachine) opcode_INST() error { return ErrOpcodeNotImplemented } /** Opcode: STOP Stop the unpickling machine. Every pickle ends with this opcode. The object at the top of the stack is popped, and that's the result of unpickling. The stack should be empty then. ** Stack before: [any] Stack after: [] **/ func (pm *PickleMachine) opcode_STOP() error { return ErrOpcodeStopped } /** Opcode: PERSID Push an object identified by a persistent ID. The pickle module doesn't define what a persistent ID means. PERSID's argument is a newline-terminated str-style (no embedded escapes, no bracketing quote characters) string, which *is* "the persistent ID". The unpickler passes this string to self.persistent_load(). Whatever object that returns is pushed on the stack. There is no implementation of persistent_load() in Python's unpickler: it must be supplied by an unpickler subclass. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_PERSID() error { return ErrOpcodeNotImplemented } stalecucumber-master/fuzz.go0000644000000000000000000000026113011632561015225 0ustar rootroot// +build gofuzz package stalecucumber import ( "bytes" ) func Fuzz(data []byte) int { if _, err := Unpickle(bytes.NewReader(data)); err != nil { return 0 } return 1 } stalecucumber-master/.gitignore0000644000000000000000000000003113011632561015663 0ustar rootrootpython/*.pyc python/*.go stalecucumber-master/helpers.go0000644000000000000000000001011513011632561015670 0ustar rootrootpackage stalecucumber import "fmt" import "math/big" /* This type is returned whenever a helper cannot convert the result of Unpickle into the desired type. */ type WrongTypeError struct { Result interface{} Request string } func (wte WrongTypeError) Error() string { return fmt.Sprintf("Unpickling returned type %T which cannot be converted to %s", wte.Result, wte.Request) } func newWrongTypeError(result interface{}, request interface{}) error { return WrongTypeError{Result: result, Request: fmt.Sprintf("%T", request)} } /* This helper attempts to convert the return value of Unpickle into a string. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func String(v interface{}, err error) (string, error) { if err != nil { return "", err } vs, ok := v.(string) if ok { return vs, nil } return "", newWrongTypeError(v, vs) } /* This helper attempts to convert the return value of Unpickle into a int64. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func Int(v interface{}, err error) (int64, error) { if err != nil { return 0, err } vi, ok := v.(int64) if ok { return vi, nil } vbi, ok := v.(*big.Int) if ok { if vbi.BitLen() <= 63 { return vbi.Int64(), nil } } return 0, newWrongTypeError(v, vi) } /* This helper attempts to convert the return value of Unpickle into a bool. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func Bool(v interface{}, err error) (bool, error) { if err != nil { return false, err } vb, ok := v.(bool) if ok { return vb, nil } return false, newWrongTypeError(v, vb) } /* This helper attempts to convert the return value of Unpickle into a *big.Int. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func Big(v interface{}, err error) (*big.Int, error) { if err != nil { return nil, err } vb, ok := v.(*big.Int) if ok { return vb, nil } return nil, newWrongTypeError(v, vb) } /* This helper attempts to convert the return value of Unpickle into a float64. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func Float(v interface{}, err error) (float64, error) { if err != nil { return 0.0, err } vf, ok := v.(float64) if ok { return vf, nil } return 0.0, newWrongTypeError(v, vf) } /* This helper attempts to convert the return value of Unpickle into a []interface{}. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func ListOrTuple(v interface{}, err error) ([]interface{}, error) { if err != nil { return nil, err } vl, ok := v.([]interface{}) if ok { return vl, nil } return nil, newWrongTypeError(v, vl) } /* This helper attempts to convert the return value of Unpickle into a map[interface{}]interface{}. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func Dict(v interface{}, err error) (map[interface{}]interface{}, error) { if err != nil { return nil, err } vd, ok := v.(map[interface{}]interface{}) if ok { return vd, nil } return nil, newWrongTypeError(v, vd) } /* This helper attempts to convert the return value of Unpickle into a map[string]interface{}. If Unpickle returns an error that error is returned immediately. If the value cannot be converted an error is returned. */ func DictString(v interface{}, err error) (map[string]interface{}, error) { var src map[interface{}]interface{} src, err = Dict(v, err) if err != nil { return nil, err } return tryDictToDictString(src) } func tryDictToDictString(src map[interface{}]interface{}) (map[string]interface{}, error) { dst := make(map[string]interface{}, len(src)) for k, v := range src { kstr, ok := k.(string) if !ok { return nil, newWrongTypeError(src, dst) } dst[kstr] = v } return dst, nil } stalecucumber-master/struct_export_test/0000755000000000000000000000000013011632561017665 5ustar rootrootstalecucumber-master/struct_export_test/structs.go0000644000000000000000000000040213011632561021717 0ustar rootrootpackage struct_export_test type struct0 struct { Joker string Killer float32 Duplicate int } type Struct1 struct { struct0 Lawnmower uint8 Duplicate float64 shouldntMessWithMe int likewise int `pickle:"foobar"` } stalecucumber-master/jump_list.go0000644000000000000000000000062213011632561016236 0ustar rootrootpackage stalecucumber import "errors" type opcodeFunc func(*PickleMachine) error var ErrOpcodeInvalid = errors.New("Opcode is invalid") func (pm *PickleMachine) Opcode_Invalid() error { return ErrOpcodeInvalid } type opcodeJumpList [256]opcodeFunc func buildEmptyJumpList() opcodeJumpList { jl := opcodeJumpList{} for i := range jl { jl[i] = (*PickleMachine).Opcode_Invalid } return jl } stalecucumber-master/fuzz_test.go0000644000000000000000000000066613011632561016275 0ustar rootrootpackage stalecucumber import ( "strings" "testing" ) func TestFuzzCrashers(t *testing.T) { var crashers = []string{ "}}(s", //protocol_0 SETITEM hash of unhashable "((d}d", //protocol_0.go opcode_DICT hash of unhashable "}(}(a}u", //protocol_1 SETITEMS hash of unhashable "(p0\nj0000", //pickle_machine flushMemoBuffer index out of range } for _, f := range crashers { Unpickle(strings.NewReader(f)) } } stalecucumber-master/tuple.go0000644000000000000000000000017613011632561015365 0ustar rootrootpackage stalecucumber type PickleTuple []interface{} func NewTuple(v ...interface{}) PickleTuple { return PickleTuple(v) } stalecucumber-master/protocol_2.go0000644000000000000000000001471313011632561016320 0ustar rootrootpackage stalecucumber import "fmt" import "math/big" /** Opcode: LONG1 (0x8a) Long integer using one-byte length. An arbitrary length integer encoded as a bytestring. A single byte following the opcode indicates the length of the bytestring If the string length is zero, then the value is zero. Otherwise the bytestring is 256-complement representation of an integer in reverse. ** Stack before: [] Stack after: [long] **/ func (pm *PickleMachine) opcode_LONG1() error { var l uint8 err := pm.readBinaryInto(&l, false) if err != nil { return err } if l == 0 { pm.push(big.NewInt(0)) return nil } reversedData, err := pm.readFixedLengthRaw(int64(l)) if err != nil { return err } //For no obvious reason, the python pickler //always reverses the bytes. Reverse it here var data [256]byte { var j int for i := len(reversedData) - 1; i != -1; i-- { data[j] = reversedData[i] j++ } } v := new(big.Int) v.SetBytes(data[:l]) invertIfNegative(data[0], v, int(l)) pm.push(v) return nil } func invertIfNegative(first byte, v *big.Int, l int) { var negative bool //Check for negative number. negative = (0x80 & first) != 0x0 if negative { offset := big.NewInt(1) offset.Lsh(offset, uint(l*8)) v.Sub(v, offset) } } /** Opcode: LONG4 (0x8b) Long integer using found-byte length. An arbitrary length integer encoded as a bytestring. A four-byte signed little-endian integer following the opcode indicates the length of the bytestring. If the string length is zero, then the value is zero. Otherwise the bytestring is 256-complement representation of an integer in reverse.** Stack before: [] Stack after: [long] **/ func (pm *PickleMachine) opcode_LONG4() error { var l uint32 err := pm.readBinaryInto(&l, false) if err != nil { return err } if l == 0 { pm.push(big.NewInt(0)) return nil } reversedData, err := pm.readFixedLengthRaw(int64(l)) if err != nil { return err } //For no obvious reason, the python pickler //always reverses the bytes. Reverse it here data := make([]byte, len(reversedData)) { var j int for i := len(reversedData) - 1; i != -1; i-- { data[j] = reversedData[i] j++ } } v := new(big.Int) v.SetBytes(data[:l]) invertIfNegative(data[0], v, len(data)) pm.push(v) return nil } /** Opcode: NEWTRUE (0x88) True. Push True onto the stack.** Stack before: [] Stack after: [bool] **/ func (pm *PickleMachine) opcode_NEWTRUE() error { pm.push(true) return nil } /** Opcode: NEWFALSE (0x89) True. Push False onto the stack.** Stack before: [] Stack after: [bool] **/ func (pm *PickleMachine) opcode_NEWFALSE() error { pm.push(false) return nil } /** Opcode: TUPLE1 (0x85) Build a one-tuple out of the topmost item on the stack. This code pops one value off the stack and pushes a tuple of length 1 whose one item is that value back onto it. In other words: stack[-1] = tuple(stack[-1:]) ** Stack before: [any] Stack after: [tuple] **/ func (pm *PickleMachine) opcode_TUPLE1() error { v, err := pm.pop() if err != nil { return err } pm.push([]interface{}{v}) return nil } /** Opcode: TUPLE2 (0x86) Build a two-tuple out of the top two items on the stack. This code pops two values off the stack and pushes a tuple of length 2 whose items are those values back onto it. In other words: stack[-2:] = [tuple(stack[-2:])] ** Stack before: [any, any] Stack after: [tuple] **/ func (pm *PickleMachine) opcode_TUPLE2() error { v := make([]interface{}, 2) var err error v[1], err = pm.pop() if err != nil { return err } v[0], err = pm.pop() if err != nil { return err } pm.push(v) return nil } /** Opcode: TUPLE3 (0x87) Build a three-tuple out of the top three items on the stack. This code pops three values off the stack and pushes a tuple of length 3 whose items are those values back onto it. In other words: stack[-3:] = [tuple(stack[-3:])] ** Stack before: [any, any, any] Stack after: [tuple] **/ func (pm *PickleMachine) opcode_TUPLE3() error { v := make([]interface{}, 3) var err error v[2], err = pm.pop() if err != nil { return err } v[1], err = pm.pop() if err != nil { return err } v[0], err = pm.pop() if err != nil { return err } pm.push(v) return nil } /** Opcode: EXT1 (0x82) Extension code. This code and the similar EXT2 and EXT4 allow using a registry of popular objects that are pickled by name, typically classes. It is envisioned that through a global negotiation and registration process, third parties can set up a mapping between ints and object names. In order to guarantee pickle interchangeability, the extension code registry ought to be global, although a range of codes may be reserved for private use. EXT1 has a 1-byte integer argument. This is used to index into the extension registry, and the object at that index is pushed on the stack. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_EXT1() error { return ErrOpcodeNotImplemented } /** Opcode: EXT2 (0x83) Extension code. See EXT1. EXT2 has a two-byte integer argument. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_EXT2() error { return ErrOpcodeNotImplemented } /** Opcode: EXT4 (0x84) Extension code. See EXT1. EXT4 has a four-byte integer argument. ** Stack before: [] Stack after: [any] **/ func (pm *PickleMachine) opcode_EXT4() error { return ErrOpcodeNotImplemented } /** Opcode: NEWOBJ (0x81) Build an object instance. The stack before should be thought of as containing a class object followed by an argument tuple (the tuple being the stack top). Call these cls and args. They are popped off the stack, and the value returned by cls.__new__(cls, *args) is pushed back onto the stack. ** Stack before: [any, any] Stack after: [any] **/ func (pm *PickleMachine) opcode_NEWOBJ() error { return ErrOpcodeNotImplemented } /** Opcode: PROTO (0x80) Protocol version indicator. For protocol 2 and above, a pickle must start with this opcode. The argument is the protocol version, an int in range(2, 256). ** Stack before: [] Stack after: [] **/ func (pm *PickleMachine) opcode_PROTO() error { var version int8 err := pm.readBinaryInto(&version, false) if err != nil { return err } if version != 2 { return fmt.Errorf("Unsupported version #%d detected", version) } return nil } stalecucumber-master/python/0000755000000000000000000000000013011632561015222 5ustar rootrootstalecucumber-master/python/doc.py0000644000000000000000000000132713011632561016344 0ustar rootrootimport pickletools import sys import textwrap import versions def print_opcode(opcode): sys.stdout.write("##Name: %s\n" % opcode.name) sys.stdout.write("* Opcode: 0x%X\n" % ord(opcode.code)) sys.stdout.write("* Stack before: `%s`\n" % opcode.stack_before) sys.stdout.write("* Stack after: `%s`\n" % opcode.stack_after) sys.stdout.write("\n") sys.stdout.write("###Description\n") sys.stdout.write(opcode.doc) sys.stdout.write("\n") sys.stdout.write("#Version 0\n") for opcode in versions.v0: print_opcode(opcode) sys.stdout.write("#Version 1\n") for opcode in versions.v1: print_opcode(opcode) sys.stdout.write("#Version 2\n") for opcode in versions.v1: print_opcode(opcode) stalecucumber-master/python/versions.py0000644000000000000000000000034313011632561017444 0ustar rootrootimport pickletools v0 = [opcode for opcode in pickletools.opcodes if opcode.proto == 0] v1 = [opcode for opcode in pickletools.opcodes if opcode.proto == 1] v2 = [opcode for opcode in pickletools.opcodes if opcode.proto == 2] stalecucumber-master/python/scaffolding.py0000644000000000000000000000314313011632561020054 0ustar rootrootimport versions from io import open import itertools pkg_stmt = u'package stalecucumber\n\n' def make_name(n): return "opcode_%s" % n def write_opcode(fout,opcode): func_name = make_name(opcode.name) fout.write(u'/**\n') fout.write(u"Opcode: %s (0x%x)\n%s**\n" % (opcode.name,ord(opcode.code),opcode.doc)) fout.write(u"Stack before: %s\n" % opcode.stack_before) fout.write(u"Stack after: %s\n" % opcode.stack_after) fout.write(u'**/\n') fout.write(u"func (pm *PickleMachine) %s () error {\n" % func_name) fout.write(u'return ErrOpcodeNotImplemented\n') fout.write(u'}\n\n') with open('protocol_0.go','w') as fout: fout.write(pkg_stmt) for opcode in versions.v0: write_opcode(fout,opcode) with open('protocol_1.go','w') as fout: fout.write(pkg_stmt) for opcode in versions.v1: write_opcode(fout,opcode) with open('protocol_2.go','w') as fout: fout.write(pkg_stmt) for opcode in versions.v2: write_opcode(fout,opcode) all_opcodes = [versions.v0,versions.v1,versions.v2] def opcode_to_const_name(o): return u'OPCODE_%s' % o.name with open('opcodes.go','w') as fout: fout.write(pkg_stmt) for opcode in itertools.chain(*all_opcodes): fout.write(u'const ') fout.write(opcode_to_const_name(opcode)) fout.write(u' = ') fout.write(u'0x%x' % ord(opcode.code)) fout.write(u'\n') with open('populate_jump_list.go','w') as fout: fout.write(pkg_stmt) fout.write(u'func populateJumpList(jl *opcodeJumpList) {\n') for opcode in itertools.chain(*all_opcodes): fout.write(u"\tjl[%s] = (*PickleMachine).%s\n" % (opcode_to_const_name(opcode),make_name(opcode.name))) fout.write(u"}\n\n") stalecucumber-master/pickle_machine_test.go0000644000000000000000000006204213011632561020226 0ustar rootrootpackage stalecucumber import "testing" import "strings" import "math/big" import "reflect" import "fmt" import "bytes" import "unicode/utf8" func testString(t *testing.T, input string, expect string) { reader := strings.NewReader(input) result, err := String(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if result != expect { t.Fatalf("Got %q(%T) expected %q(%T)", result, result, expect, expect) } } func TestJorilxUnicodeString(t *testing.T) { input := []byte{0x56, 0xE0, 0x0A, // Và 0x70, 0x30, 0x0A, // p0 0x2E} // . reader := bytes.NewReader(input) unpickled, err := String(Unpickle(reader)) const EXPECT = `à` if err != nil { t.Fatal(err) } if utf8.RuneCountInString(unpickled) != 1 { t.Errorf("wrong length string unpacked %d,%q", utf8.RuneCountInString(unpickled), unpickled) } if unpickled != EXPECT { t.Errorf("Expected %q got %q", EXPECT, unpickled) } inputProtocol2 := strings.NewReader("\x80\x02X\x02\x00\x00\x00\xc3\xa0q\x00.") unpickled, err = String(Unpickle(inputProtocol2)) if err != nil { t.Fatal(err) } if utf8.RuneCountInString(unpickled) != 1 { t.Errorf("wrong length string unpacked %d,%q", utf8.RuneCountInString(unpickled), unpickled) } if unpickled != EXPECT { t.Errorf("Expected %q got %q", EXPECT, unpickled) } const EXPECT_SNOWMAN = "à ☃" const inputWithSnowman = "\x56\xe0\x20\x5c\x75\x32\x36\x30\x33\x0a\x70\x30\x0a\x2e" unpickled, err = String(Unpickle(strings.NewReader(inputWithSnowman))) if err != nil { t.Fatal(err) } if unpickled != EXPECT_SNOWMAN { t.Errorf("Expected %q got %q", EXPECT_SNOWMAN, unpickled) } } func TestProtocol0Integer(t *testing.T) { reader := strings.NewReader("I42\n.") result, err := Int(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } const EXPECT = 42 if result != EXPECT { t.Fatalf("Got value %d expected %d", result, EXPECT) } } func TestProtocol0Bool(t *testing.T) { var result bool reader := strings.NewReader("I00\n.") result, err := Bool(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if result != false { t.Fatalf("Got value %v expected %v", result, false) } reader = strings.NewReader("I01\n.") result, err = Bool(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if result != true { t.Fatalf("Got value %v expected %v", result, true) } } func TestProtocol0String(t *testing.T) { testString(t, "S''\np0\n.", "") testString(t, "S'foobar'\np0\n.", "foobar") testString(t, "S'String with embedded\\nnewline.'\np0\n.", "String with embedded\nnewline.") testString(t, "\x53\x27\x53\x74\x72\x69\x6e\x67\x20\x77\x69\x74\x68\x20\x65\x6d\x62\x65\x64\x64\x65\x64\x5c\x6e\x6e\x65\x77\x6c\x69\x6e\x65\x20\x61\x6e\x64\x20\x65\x6d\x62\x65\x64\x64\x65\x64\x20\x71\x75\x6f\x74\x65\x20\x5c\x27\x20\x61\x6e\x64\x20\x65\x6d\x62\x65\x64\x64\x65\x64\x20\x64\x6f\x75\x62\x6c\x65\x71\x75\x6f\x74\x65\x20\x22\x2e\x27\x0a\x70\x30\x0a\x2e", "String with embedded\nnewline and embedded quote ' and embedded doublequote \".") } func testBigIntFromString(t *testing.T, input string, expectStr string) { var expect big.Int _, err := fmt.Sscanf(expectStr, "%d", &expect) if err != nil { t.Fatalf("got error parsing %q:%v", expectStr, err) } testBigInt(t, input, &expect) } func testBigInt(t *testing.T, input string, expect *big.Int) { reader := strings.NewReader(input) result, err := Big(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if result.Cmp(expect) != 0 { t.Fatalf("Got value %s expected %s", result, expect) } } func TestProtocol0Long(t *testing.T) { testBigInt(t, "L5L\n.", big.NewInt(5)) testBigIntFromString(t, "L18446744073709551615L\n.", "18446744073709551615") testBigIntFromString(t, "L-18446744073709551615L\n.", "-18446744073709551615") } func TestProtocol0Float(t *testing.T) { reader := strings.NewReader("F3.14\n.") const EXPECT = 3.14 result, err := Float(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if EXPECT != result { t.Fatalf("Got value %q expected %q", result, EXPECT) } } func testDict(t *testing.T, input string, expect map[interface{}]interface{}) { reader := strings.NewReader(input) result, err := Dict(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if len(result) != len(expect) { t.Errorf("result has wrong length %d", len(result)) } for k, v := range result { var expectedV interface{} expectedV, ok := expect[k] if !ok { t.Errorf("Result has key %v(%T) which is not in expectation", k, k) continue } if reflect.TypeOf(v) != reflect.TypeOf(expectedV) { t.Errorf("At key %v result has type %T where expectation has type %T", k, v, expectedV) continue } if !reflect.DeepEqual(expectedV, v) { t.Errorf("At key %v result %v != expectation %v", k, v, expectedV) } } } func TestProtocol0Get(t *testing.T) { testList(t, "(lp0\nS'hydrogen18'\np1\nag1\na.", []interface{}{"hydrogen18", "hydrogen18"}) } func TestProtocol1Get(t *testing.T) { testList(t, "]q\x00(U\nhydrogen18q\x01h\x01e.", []interface{}{"hydrogen18", "hydrogen18"}) } func TestProtocol0Dict(t *testing.T) { { input := "(dp0\nS'a'\np1\nI1\nsS'b'\np2\nI5\ns." expect := make(map[interface{}]interface{}) expect["a"] = int64(1) expect["b"] = int64(5) testDict(t, input, expect) } { expect := make(map[interface{}]interface{}) expect["foo"] = "bar" expect[int64(5)] = "kitty" expect["num"] = 13.37 expect["list"] = []interface{}{int64(1), int64(2), int64(3), int64(4)} testDict(t, "(dp0\nS'list'\np1\n(lp2\nI1\naI2\naI3\naI4\nasS'foo'\np3\nS'bar'\np4\nsS'num'\np5\nF13.37\nsI5\nS'kitty'\np6\ns.", expect) } } func TestProtocol1Dict(t *testing.T) { testDict(t, "}q\x00.", make(map[interface{}]interface{})) { expect := make(map[interface{}]interface{}) expect["foo"] = "bar" expect["meow"] = "bar" expect[int64(5)] = "kitty" expect["num"] = 13.37 expect["list"] = []interface{}{int64(1), int64(2), int64(3), int64(4)} input := "}q\x00(U\x04meowq\x01U\x03barq\x02U\x04listq\x03]q\x04(K\x01K\x02K\x03K\x04eU\x03fooq\x05h\x02U\x03numq\x06G@*\xbdp\xa3\xd7\n=K\x05U\x05kittyq\x07u." testDict(t, input, expect) } } func testListsEqual(t *testing.T, result []interface{}, expect []interface{}) { if len(result) != len(expect) { t.Errorf("Result has wrong length %d", len(result)) } for i, v := range result { vexpect := expect[i] if !reflect.DeepEqual(v, vexpect) { t.Errorf("result[%v](%T) != expect[%v](%T)", i, v, i, vexpect) t.Errorf("result[%d]=%v", i, v) t.Errorf("expect[%d]=%v", i, vexpect) } } } func testList(t *testing.T, input string, expect []interface{}) { reader := strings.NewReader(input) result, err := ListOrTuple(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } testListsEqual(t, result, expect) } func TestProtocol0List(t *testing.T) { testList(t, "(lp0\nI1\naI2\naI3\na.", []interface{}{int64(1), int64(2), int64(3)}) } func TestProtocol1List(t *testing.T) { testList(t, "]q\x00.", []interface{}{}) testList(t, "]q\x00(M9\x05M9\x05M9\x05e.", []interface{}{int64(1337), int64(1337), int64(1337)}) testList(t, "]q\x00(M9\x05I3735928559\nM\xb1\"e.", []interface{}{int64(1337), int64(0xdeadbeef), int64(8881)}) } func TestProtocol1Tuple(t *testing.T) { testList(t, ").", []interface{}{}) testList(t, "(K*K\x18K*K\x1cKRK\x1ctq\x00.", []interface{}{int64(42), int64(24), int64(42), int64(28), int64(82), int64(28)}) } func testInt(t *testing.T, input string, expect int64) { reader := strings.NewReader(input) result, err := Int(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } if result != expect { t.Fatalf("Got %d(%T) expected %d(%T)", result, result, expect, expect) } } func TestProtocol1Binint(t *testing.T) { testInt(t, "J\xff\xff\xff\x00.", 0xffffff) testInt(t, "K*.", 42) testInt(t, "M\xff\xab.", 0xabff) } func TestProtocol1String(t *testing.T) { testString(t, "U\x00q\x00.", "") testString(t, "T\x04\x01\x00\x00abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZq\x00.", "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") testString(t, "U\x13queen of the castleq\x00.", "queen of the castle") } func TestProtocol1Float(t *testing.T) { reader := strings.NewReader("G?\xc1\x1d\x14\xe3\xbc\xd3[.") result, err := Float(Unpickle(reader)) if err != nil { t.Fatalf("Got error %v", err) } var expect float64 expect = 0.1337 if result != expect { t.Fatalf("Got %f expected %f", result, expect) } } func TestProtocol1PopMark(t *testing.T) { /** This exapmle is ultra-contrived. I could not get anything to produce usage of POP_MARK using protocol 1. There are some comments in Lib/pickle.py about a recursive tuple generating this but I have no idea how that is even possible. The disassembly of this is 0: K BININT1 1 2: ( MARK 3: K BININT1 2 5: K BININT1 3 7: 1 POP_MARK (MARK at 2) 8: . STOP There is just a mark placed on the stack with some numbers afterwards solely to test the correct behavior of the POP_MARK instruction. **/ reader := strings.NewReader("K\x01(K\x02K\x031.") result, err := Int(Unpickle(reader)) const EXPECT = 1 if err != nil { t.Fatalf("Got error %v", err) } if EXPECT != result { t.Fatalf("Got %d expected %d", result, EXPECT) } } func TestProtocol1Unicode(t *testing.T) { testString(t, "X\x00\x00\x00\x00q\x00.", "") expect := "This is a slash \\. This is a newline \n. This is a character that is two embedded newlines: \u0a0a. This is a snowman: \u2603." if len([]rune(expect)) != 115 { t.Errorf("Expect shouldn't be :%v", expect) t.Fatalf("you messed up the escape sequence on the expecation, again. Length is %d", len(expect)) } testString(t, "\x58\x77\x00\x00\x00\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x73\x6c\x61\x73\x68\x20\x5c\x2e\x20\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x6e\x65\x77\x6c\x69\x6e\x65\x20\x0a\x2e\x20\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x63\x68\x61\x72\x61\x63\x74\x65\x72\x20\x74\x68\x61\x74\x20\x69\x73\x20\x74\x77\x6f\x20\x65\x6d\x62\x65\x64\x64\x65\x64\x20\x6e\x65\x77\x6c\x69\x6e\x65\x73\x3a\x20\xe0\xa8\x8a\x2e\x20\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x73\x6e\x6f\x77\x6d\x61\x6e\x3a\x20\xe2\x98\x83\x2e\x71\x00\x2e", expect) } func TestProtocol0Unicode(t *testing.T) { testString(t, "V\np0\n.", "") expect := "This is a slash \\. This is a newline \n. This is a character that is two embedded newlines: \u0a0a. This is a snowman: \u2603." if len([]rune(expect)) != 115 { t.Errorf("Expect shouldn't be :%v", expect) t.Fatalf("you messed up the escape sequence on the expecation, again. Length is %d", len(expect)) } testString(t, "\x56\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x73\x6c\x61\x73\x68\x20\x5c\x75\x30\x30\x35\x63\x2e\x20\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x6e\x65\x77\x6c\x69\x6e\x65\x20\x5c\x75\x30\x30\x30\x61\x2e\x20\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x63\x68\x61\x72\x61\x63\x74\x65\x72\x20\x74\x68\x61\x74\x20\x69\x73\x20\x74\x77\x6f\x20\x65\x6d\x62\x65\x64\x64\x65\x64\x20\x6e\x65\x77\x6c\x69\x6e\x65\x73\x3a\x20\x5c\x75\x30\x61\x30\x61\x2e\x20\x54\x68\x69\x73\x20\x69\x73\x20\x61\x20\x73\x6e\x6f\x77\x6d\x61\x6e\x3a\x20\x5c\x75\x32\x36\x30\x33\x2e\x0a\x70\x30\x0a\x2e", expect) } func TestProtocol1LongBinPutBinGet(t *testing.T) { input := "\x5d\x71\x00\x28\x5d\x71\x01\x28\x55\x01\x30\x71\x02\x55\x01\x31\x71\x03\x55\x01\x32\x71\x04\x55\x01\x33\x71\x05\x55\x01\x34\x71\x06\x55\x01\x35\x71\x07\x55\x01\x36\x71\x08\x55\x01\x37\x71\x09\x55\x01\x38\x71\x0a\x55\x01\x39\x71\x0b\x55\x02\x31\x30\x71\x0c\x55\x02\x31\x31\x71\x0d\x55\x02\x31\x32\x71\x0e\x55\x02\x31\x33\x71\x0f\x55\x02\x31\x34\x71\x10\x55\x02\x31\x35\x71\x11\x55\x02\x31\x36\x71\x12\x55\x02\x31\x37\x71\x13\x55\x02\x31\x38\x71\x14\x55\x02\x31\x39\x71\x15\x55\x02\x32\x30\x71\x16\x55\x02\x32\x31\x71\x17\x55\x02\x32\x32\x71\x18\x55\x02\x32\x33\x71\x19\x55\x02\x32\x34\x71\x1a\x55\x02\x32\x35\x71\x1b\x55\x02\x32\x36\x71\x1c\x55\x02\x32\x37\x71\x1d\x55\x02\x32\x38\x71\x1e\x55\x02\x32\x39\x71\x1f\x55\x02\x33\x30\x71\x20\x55\x02\x33\x31\x71\x21\x55\x02\x33\x32\x71\x22\x55\x02\x33\x33\x71\x23\x55\x02\x33\x34\x71\x24\x55\x02\x33\x35\x71\x25\x55\x02\x33\x36\x71\x26\x55\x02\x33\x37\x71\x27\x55\x02\x33\x38\x71\x28\x55\x02\x33\x39\x71\x29\x55\x02\x34\x30\x71\x2a\x55\x02\x34\x31\x71\x2b\x55\x02\x34\x32\x71\x2c\x55\x02\x34\x33\x71\x2d\x55\x02\x34\x34\x71\x2e\x55\x02\x34\x35\x71\x2f\x55\x02\x34\x36\x71\x30\x55\x02\x34\x37\x71\x31\x55\x02\x34\x38\x71\x32\x55\x02\x34\x39\x71\x33\x55\x02\x35\x30\x71\x34\x55\x02\x35\x31\x71\x35\x55\x02\x35\x32\x71\x36\x55\x02\x35\x33\x71\x37\x55\x02\x35\x34\x71\x38\x55\x02\x35\x35\x71\x39\x55\x02\x35\x36\x71\x3a\x55\x02\x35\x37\x71\x3b\x55\x02\x35\x38\x71\x3c\x55\x02\x35\x39\x71\x3d\x55\x02\x36\x30\x71\x3e\x55\x02\x36\x31\x71\x3f\x55\x02\x36\x32\x71\x40\x55\x02\x36\x33\x71\x41\x55\x02\x36\x34\x71\x42\x55\x02\x36\x35\x71\x43\x55\x02\x36\x36\x71\x44\x55\x02\x36\x37\x71\x45\x55\x02\x36\x38\x71\x46\x55\x02\x36\x39\x71\x47\x55\x02\x37\x30\x71\x48\x55\x02\x37\x31\x71\x49\x55\x02\x37\x32\x71\x4a\x55\x02\x37\x33\x71\x4b\x55\x02\x37\x34\x71\x4c\x55\x02\x37\x35\x71\x4d\x55\x02\x37\x36\x71\x4e\x55\x02\x37\x37\x71\x4f\x55\x02\x37\x38\x71\x50\x55\x02\x37\x39\x71\x51\x55\x02\x38\x30\x71\x52\x55\x02\x38\x31\x71\x53\x55\x02\x38\x32\x71\x54\x55\x02\x38\x33\x71\x55\x55\x02\x38\x34\x71\x56\x55\x02\x38\x35\x71\x57\x55\x02\x38\x36\x71\x58\x55\x02\x38\x37\x71\x59\x55\x02\x38\x38\x71\x5a\x55\x02\x38\x39\x71\x5b\x55\x02\x39\x30\x71\x5c\x55\x02\x39\x31\x71\x5d\x55\x02\x39\x32\x71\x5e\x55\x02\x39\x33\x71\x5f\x55\x02\x39\x34\x71\x60\x55\x02\x39\x35\x71\x61\x55\x02\x39\x36\x71\x62\x55\x02\x39\x37\x71\x63\x55\x02\x39\x38\x71\x64\x55\x02\x39\x39\x71\x65\x55\x03\x31\x30\x30\x71\x66\x55\x03\x31\x30\x31\x71\x67\x55\x03\x31\x30\x32\x71\x68\x55\x03\x31\x30\x33\x71\x69\x55\x03\x31\x30\x34\x71\x6a\x55\x03\x31\x30\x35\x71\x6b\x55\x03\x31\x30\x36\x71\x6c\x55\x03\x31\x30\x37\x71\x6d\x55\x03\x31\x30\x38\x71\x6e\x55\x03\x31\x30\x39\x71\x6f\x55\x03\x31\x31\x30\x71\x70\x55\x03\x31\x31\x31\x71\x71\x55\x03\x31\x31\x32\x71\x72\x55\x03\x31\x31\x33\x71\x73\x55\x03\x31\x31\x34\x71\x74\x55\x03\x31\x31\x35\x71\x75\x55\x03\x31\x31\x36\x71\x76\x55\x03\x31\x31\x37\x71\x77\x55\x03\x31\x31\x38\x71\x78\x55\x03\x31\x31\x39\x71\x79\x55\x03\x31\x32\x30\x71\x7a\x55\x03\x31\x32\x31\x71\x7b\x55\x03\x31\x32\x32\x71\x7c\x55\x03\x31\x32\x33\x71\x7d\x55\x03\x31\x32\x34\x71\x7e\x55\x03\x31\x32\x35\x71\x7f\x55\x03\x31\x32\x36\x71\x80\x55\x03\x31\x32\x37\x71\x81\x55\x03\x31\x32\x38\x71\x82\x55\x03\x31\x32\x39\x71\x83\x55\x03\x31\x33\x30\x71\x84\x55\x03\x31\x33\x31\x71\x85\x55\x03\x31\x33\x32\x71\x86\x55\x03\x31\x33\x33\x71\x87\x55\x03\x31\x33\x34\x71\x88\x55\x03\x31\x33\x35\x71\x89\x55\x03\x31\x33\x36\x71\x8a\x55\x03\x31\x33\x37\x71\x8b\x55\x03\x31\x33\x38\x71\x8c\x55\x03\x31\x33\x39\x71\x8d\x55\x03\x31\x34\x30\x71\x8e\x55\x03\x31\x34\x31\x71\x8f\x55\x03\x31\x34\x32\x71\x90\x55\x03\x31\x34\x33\x71\x91\x55\x03\x31\x34\x34\x71\x92\x55\x03\x31\x34\x35\x71\x93\x55\x03\x31\x34\x36\x71\x94\x55\x03\x31\x34\x37\x71\x95\x55\x03\x31\x34\x38\x71\x96\x55\x03\x31\x34\x39\x71\x97\x55\x03\x31\x35\x30\x71\x98\x55\x03\x31\x35\x31\x71\x99\x55\x03\x31\x35\x32\x71\x9a\x55\x03\x31\x35\x33\x71\x9b\x55\x03\x31\x35\x34\x71\x9c\x55\x03\x31\x35\x35\x71\x9d\x55\x03\x31\x35\x36\x71\x9e\x55\x03\x31\x35\x37\x71\x9f\x55\x03\x31\x35\x38\x71\xa0\x55\x03\x31\x35\x39\x71\xa1\x55\x03\x31\x36\x30\x71\xa2\x55\x03\x31\x36\x31\x71\xa3\x55\x03\x31\x36\x32\x71\xa4\x55\x03\x31\x36\x33\x71\xa5\x55\x03\x31\x36\x34\x71\xa6\x55\x03\x31\x36\x35\x71\xa7\x55\x03\x31\x36\x36\x71\xa8\x55\x03\x31\x36\x37\x71\xa9\x55\x03\x31\x36\x38\x71\xaa\x55\x03\x31\x36\x39\x71\xab\x55\x03\x31\x37\x30\x71\xac\x55\x03\x31\x37\x31\x71\xad\x55\x03\x31\x37\x32\x71\xae\x55\x03\x31\x37\x33\x71\xaf\x55\x03\x31\x37\x34\x71\xb0\x55\x03\x31\x37\x35\x71\xb1\x55\x03\x31\x37\x36\x71\xb2\x55\x03\x31\x37\x37\x71\xb3\x55\x03\x31\x37\x38\x71\xb4\x55\x03\x31\x37\x39\x71\xb5\x55\x03\x31\x38\x30\x71\xb6\x55\x03\x31\x38\x31\x71\xb7\x55\x03\x31\x38\x32\x71\xb8\x55\x03\x31\x38\x33\x71\xb9\x55\x03\x31\x38\x34\x71\xba\x55\x03\x31\x38\x35\x71\xbb\x55\x03\x31\x38\x36\x71\xbc\x55\x03\x31\x38\x37\x71\xbd\x55\x03\x31\x38\x38\x71\xbe\x55\x03\x31\x38\x39\x71\xbf\x55\x03\x31\x39\x30\x71\xc0\x55\x03\x31\x39\x31\x71\xc1\x55\x03\x31\x39\x32\x71\xc2\x55\x03\x31\x39\x33\x71\xc3\x55\x03\x31\x39\x34\x71\xc4\x55\x03\x31\x39\x35\x71\xc5\x55\x03\x31\x39\x36\x71\xc6\x55\x03\x31\x39\x37\x71\xc7\x55\x03\x31\x39\x38\x71\xc8\x55\x03\x31\x39\x39\x71\xc9\x55\x03\x32\x30\x30\x71\xca\x55\x03\x32\x30\x31\x71\xcb\x55\x03\x32\x30\x32\x71\xcc\x55\x03\x32\x30\x33\x71\xcd\x55\x03\x32\x30\x34\x71\xce\x55\x03\x32\x30\x35\x71\xcf\x55\x03\x32\x30\x36\x71\xd0\x55\x03\x32\x30\x37\x71\xd1\x55\x03\x32\x30\x38\x71\xd2\x55\x03\x32\x30\x39\x71\xd3\x55\x03\x32\x31\x30\x71\xd4\x55\x03\x32\x31\x31\x71\xd5\x55\x03\x32\x31\x32\x71\xd6\x55\x03\x32\x31\x33\x71\xd7\x55\x03\x32\x31\x34\x71\xd8\x55\x03\x32\x31\x35\x71\xd9\x55\x03\x32\x31\x36\x71\xda\x55\x03\x32\x31\x37\x71\xdb\x55\x03\x32\x31\x38\x71\xdc\x55\x03\x32\x31\x39\x71\xdd\x55\x03\x32\x32\x30\x71\xde\x55\x03\x32\x32\x31\x71\xdf\x55\x03\x32\x32\x32\x71\xe0\x55\x03\x32\x32\x33\x71\xe1\x55\x03\x32\x32\x34\x71\xe2\x55\x03\x32\x32\x35\x71\xe3\x55\x03\x32\x32\x36\x71\xe4\x55\x03\x32\x32\x37\x71\xe5\x55\x03\x32\x32\x38\x71\xe6\x55\x03\x32\x32\x39\x71\xe7\x55\x03\x32\x33\x30\x71\xe8\x55\x03\x32\x33\x31\x71\xe9\x55\x03\x32\x33\x32\x71\xea\x55\x03\x32\x33\x33\x71\xeb\x55\x03\x32\x33\x34\x71\xec\x55\x03\x32\x33\x35\x71\xed\x55\x03\x32\x33\x36\x71\xee\x55\x03\x32\x33\x37\x71\xef\x55\x03\x32\x33\x38\x71\xf0\x55\x03\x32\x33\x39\x71\xf1\x55\x03\x32\x34\x30\x71\xf2\x55\x03\x32\x34\x31\x71\xf3\x55\x03\x32\x34\x32\x71\xf4\x55\x03\x32\x34\x33\x71\xf5\x55\x03\x32\x34\x34\x71\xf6\x55\x03\x32\x34\x35\x71\xf7\x55\x03\x32\x34\x36\x71\xf8\x55\x03\x32\x34\x37\x71\xf9\x55\x03\x32\x34\x38\x71\xfa\x55\x03\x32\x34\x39\x71\xfb\x55\x03\x32\x35\x30\x71\xfc\x55\x03\x32\x35\x31\x71\xfd\x55\x03\x32\x35\x32\x71\xfe\x55\x03\x32\x35\x33\x71\xff\x55\x03\x32\x35\x34\x72\x00\x01\x00\x00\x55\x03\x32\x35\x35\x72\x01\x01\x00\x00\x65\x55\x04\x6d\x65\x6f\x77\x72\x02\x01\x00\x00\x4b\x05\x6a\x02\x01\x00\x00\x65\x2e" expect := make([]interface{}, 4) expect[1] = "meow" expect[2] = int64(5) expect[3] = "meow" countingList := make([]interface{}, 256) for i := 0; i != len(countingList); i++ { countingList[i] = fmt.Sprintf("%d", i) } expect[0] = countingList testList(t, input, expect) } func TestProtocol2Long(t *testing.T) { testBigInt(t, "\x80\x02\x8a\x00.", big.NewInt(0)) testBigInt(t, "\x80\x02\x8a\x01\x01.", big.NewInt(1)) testBigIntFromString(t, "\x80\x02\x8a\x0bR\xd3?\xd8\x9cY\xa5\xa7_\xc9\x04.", "5786663462362423463236434") testBigIntFromString(t, "\x80\x02\x8a\x0b\xae,\xc0'c\xa6ZX\xa06\xfb.", "-5786663462362423463236434") testBigInt(t, "\x80\x02\x8a\x01\xff.", big.NewInt(-1)) testBigIntFromString(t, "\x80\x02\x8a\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff.", "-340282366920938463463374607431768211456") testBigIntFromString(t, "\x80\x02\x8a\t\xff\xff\xff\xff\xff\xff\xff\xff\x00.", "18446744073709551615") testBigIntFromString(t, "\x80\x02\x8a\t\x01\x00\x00\x00\x00\x00\x00\x00\xff.", "-18446744073709551615") //tests the long4 opcode testBigIntFromString(t, "\x80\x02\x8b\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01.", "32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230656") testBigIntFromString(t, "\x80\x02\x8b\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff.", "-32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230656") testBigIntFromString(t, "\x80\x02\x8b\x01\x01\x00\x00*\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff.", "-32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230614") } func TestProtocol2TrueFalse(t *testing.T) { result, err := Bool(Unpickle(strings.NewReader("\x80\x02\x88."))) if err != nil { t.Fatal(err) } if result != true { t.Fatal("didnt get true") } result, err = Bool(Unpickle(strings.NewReader("\x80\x02\x89."))) if err != nil { t.Fatal(err) } if result != false { t.Fatal("didnt get false") } } func TestProtocol2Tuples(t *testing.T) { testList(t, "\x80\x02N\x85q\x00.", []interface{}{PickleNone{}}) testList(t, "\x80\x02U\x05kittyq\x00K7\x86q\x01.", []interface{}{"kitty", int64(55)}) testList(t, "\x80\x02U\x05kittyq\x00K7G@*\xbdp\xa3\xd7\n=\x87q\x01.", []interface{}{"kitty", int64(55), 13.37}) } stalecucumber-master/pickle_writer.go0000644000000000000000000002360713011632561017103 0ustar rootrootpackage stalecucumber import "io" import "reflect" import "errors" import "encoding/binary" import "fmt" import "math/big" type pickleProxy interface { WriteTo(io.Writer) (int, error) } type Pickler struct { W io.Writer program []pickleProxy } /* This type is used to pickle data.Picklers are created by calling NewPickler. Each call to Pickle writes a complete pickle program to the underlying io.Writer object. Its safe to assign W to other values in between calls to Pickle. Failures return the underlying error or an instance of PicklingError. Data is always written using Pickle Protocol 2. This format is compatible with Python 2.3 and all newer version. Type Conversions Type conversion from Go types to Python types is as follows uint8,uint16,int8,int16,int32 -> Python int int,int64,uint,uint64 -> Python int if it fits, otherwise Python Long string -> Python unicode slices, arrays -> Python list maps -> Python dict bool -> Python True and False big.Int -> Python Long struct -> Python dict Structs are pickled using their field names unless a tag is present on the field specifying the name. For example type MyType struct { FirstField int SecondField int `pickle:"meow"` } This struct would be pickled into a dictionary with two keys: "FirstField" and "meow". Embedded structs are marshalled as a nested dictionary. Exported types are never pickled. Pickling Tuples There is no equivalent type to Python's tuples in Go. You may not need to use tuples at all. For example, consider the following Python code a, b, c = pickle.load(data_in) This code tries to set to the variables "a", "b", and "c" from the result of unpickling. In this case it does not matter if the source type is a Python list or a Python tuple. If you really need to write tuples, call NewTuple and pass the data in as the arguments. This special type exists to inform stalecucumber.Pickle that a tuple should be pickled. */ func NewPickler(writer io.Writer) *Pickler { retval := &Pickler{} retval.W = writer return retval } func (p *Pickler) Pickle(v interface{}) (int, error) { if p.program != nil { p.program = p.program[0:0] } err := p.dump(v) if err != nil { return 0, err } return p.writeProgram() } var programStart = []uint8{OPCODE_PROTO, 0x2} var programEnd = []uint8{OPCODE_STOP} func (p *Pickler) writeProgram() (n int, err error) { n, err = p.W.Write(programStart) if err != nil { return } var m int for _, proxy := range p.program { m, err = proxy.WriteTo(p.W) if err != nil { return } n += m } m, err = p.W.Write(programEnd) if err != nil { return } n += m return } const BININT_MAX = (1 << 31) - 1 const BININT_MIN = 0 - BININT_MAX var ErrTypeNotPickleable = errors.New("Can't pickle this type") var ErrEmptyInterfaceNotPickleable = errors.New("The empty interface is not pickleable") type PicklingError struct { V interface{} Err error } func (pe PicklingError) Error() string { return fmt.Sprintf("Failed pickling (%T)%v:%v", pe.V, pe.V, pe.Err) } func (p *Pickler) dump(input interface{}) error { if input == nil { p.pushOpcode(OPCODE_NONE) return nil } switch input := input.(type) { case int: if input <= BININT_MAX && input >= BININT_MIN { p.dumpInt(int64(input)) return nil } p.dumpIntAsLong(int64(input)) return nil case int64: if input <= BININT_MAX && input >= BININT_MIN { p.dumpInt(input) return nil } p.dumpIntAsLong(input) return nil case int8: p.dumpInt(int64(input)) return nil case int16: p.dumpInt(int64(input)) return nil case int32: p.dumpInt(int64(input)) return nil case uint8: p.dumpInt(int64(input)) return nil case uint16: p.dumpInt(int64(input)) return nil case uint32: if input <= BININT_MAX { p.dumpInt(int64(input)) return nil } p.dumpUintAsLong(uint64(input)) return nil case uint: if input <= BININT_MAX { p.dumpInt(int64(input)) return nil } p.dumpUintAsLong(uint64(input)) return nil case uint64: if input <= BININT_MAX { p.dumpInt(int64(input)) return nil } p.dumpUintAsLong(input) return nil case float32: p.dumpFloat(float64(input)) return nil case float64: p.dumpFloat(input) return nil case string: p.dumpString(input) return nil case bool: p.dumpBool(input) return nil case big.Int: p.dumpBigInt(input) return nil case PickleNone: p.pushOpcode(OPCODE_NONE) return nil case PickleTuple: l := len(input) switch l { case 0: p.pushOpcode(OPCODE_EMPTY_TUPLE) return nil case 1, 2, 3: default: p.pushOpcode(OPCODE_MARK) } for _, v := range input { err := p.dump(v) if err != nil { return err } } switch l { case 1: p.pushOpcode(OPCODE_TUPLE1) case 2: p.pushOpcode(OPCODE_TUPLE2) case 3: p.pushOpcode(OPCODE_TUPLE3) default: p.pushOpcode(OPCODE_TUPLE) } return nil } v := reflect.ValueOf(input) vKind := v.Kind() switch vKind { //Check for pointers. They can't be //meaningfully written as a pickle unless nil. Dereference //and recurse. case reflect.Ptr: if v.IsNil() { p.pushOpcode(OPCODE_NONE) return nil } return p.dump(v.Elem().Interface()) case reflect.Map: p.pushOpcode(OPCODE_EMPTY_DICT) p.pushOpcode(OPCODE_MARK) keys := v.MapKeys() for _, key := range keys { err := p.dump(key.Interface()) if err != nil { return err } val := v.MapIndex(key) err = p.dump(val.Interface()) if err != nil { return err } } p.pushOpcode(OPCODE_SETITEMS) return nil case reflect.Slice, reflect.Array: p.pushOpcode(OPCODE_EMPTY_LIST) p.pushOpcode(OPCODE_MARK) for i := 0; i != v.Len(); i++ { element := v.Index(i) p.dump(element.Interface()) } p.pushOpcode(OPCODE_APPENDS) return nil case reflect.Struct: return p.dumpStruct(v, false) } return PicklingError{V: input, Err: ErrTypeNotPickleable} } func (p *Pickler) dumpBool(v bool) { if v { p.pushOpcode(OPCODE_NEWTRUE) } else { p.pushOpcode(OPCODE_NEWFALSE) } } func (p *Pickler) dumpStruct(v reflect.Value, nested bool) error { vType := v.Type() if !nested { p.pushOpcode(OPCODE_EMPTY_DICT) p.pushOpcode(OPCODE_MARK) } for i := 0; i != v.NumField(); i++ { field := vType.Field(i) //Never attempt to write //unexported names if len(field.PkgPath) != 0 { //Check for embedded field, which can possibly be dumped if field.Anonymous { err := p.dumpStruct(v.Field(i), true) if err != nil { return err } } continue } //Prefer the tagged name of the //field, fall back to fields actual name fieldKey := field.Tag.Get(PICKLE_TAG) if len(fieldKey) == 0 { fieldKey = field.Name } p.dumpString(fieldKey) fieldValue := v.Field(i) err := p.dump(fieldValue.Interface()) if err != nil { return err } } if !nested { p.pushOpcode(OPCODE_SETITEMS) } return nil } func (p *Pickler) pushProxy(proxy pickleProxy) { p.program = append(p.program, proxy) } func (p *Pickler) dumpFloat(v float64) { p.pushProxy(floatProxy(v)) } type opcodeProxy uint8 func (proxy opcodeProxy) WriteTo(w io.Writer) (int, error) { return w.Write([]byte{byte(proxy)}) } func (p *Pickler) pushOpcode(code uint8) { p.pushProxy(opcodeProxy(code)) } type bigIntProxy struct { v *big.Int } var zeroPad = []byte{0} var maxPad = []byte{0xff} func (proxy bigIntProxy) WriteTo(w io.Writer) (int, error) { var negative = proxy.v.Sign() == -1 var raw []byte if negative { offset := big.NewInt(1) bitLen := uint(proxy.v.BitLen()) remainder := bitLen % 8 bitLen += 8 - remainder offset.Lsh(offset, bitLen) offset.Add(proxy.v, offset) raw = offset.Bytes() } else { raw = proxy.v.Bytes() } var pad []byte var padL int var highBitSet = (len(raw) > 0 && (raw[0] & 0x80) == 0x80) if negative && !highBitSet { pad = maxPad padL = 1 } else if !negative && highBitSet { pad = zeroPad padL = 1 } l := len(raw) var header interface{} if l < 256 { header = struct { Opcode uint8 Length uint8 }{ OPCODE_LONG1, uint8(l + padL), } } else { header = struct { Opcode uint8 Length uint32 }{ OPCODE_LONG4, uint32(l + padL), } } err := binary.Write(w, binary.LittleEndian, header) if err != nil { return 0, err } n := binary.Size(header) n += l n += padL reversed := make([]byte, l) for i, v := range raw { reversed[l-i-1] = v } _, err = w.Write(reversed) if err != nil { return n, err } _, err = w.Write(pad) return n, err } func (p *Pickler) dumpIntAsLong(v int64) { p.pushProxy(bigIntProxy{big.NewInt(v)}) } func (p *Pickler) dumpBigInt(v big.Int) { p.pushProxy(bigIntProxy{&v}) //Note that this is a shallow copy } func (p *Pickler) dumpUintAsLong(v uint64) { w := big.NewInt(0) w.SetUint64(v) p.pushProxy(bigIntProxy{w}) } type floatProxy float64 func (proxy floatProxy) WriteTo(w io.Writer) (int, error) { data := struct { Opcode uint8 V float64 }{ OPCODE_BINFLOAT, float64(proxy), } return binary.Size(data), binary.Write(w, binary.BigEndian, data) } type intProxy int32 func (proxy intProxy) WriteTo(w io.Writer) (int, error) { data := struct { Opcode uint8 V int32 }{ OPCODE_BININT, int32(proxy), } return binary.Size(data), binary.Write(w, binary.LittleEndian, data) } func (p *Pickler) dumpInt(v int64) { p.pushProxy(intProxy(v)) } type stringProxy string func (proxy stringProxy) V() interface{} { return proxy } func (proxy stringProxy) WriteTo(w io.Writer) (int, error) { header := struct { Opcode uint8 Length int32 }{ OPCODE_BINUNICODE, int32(len(proxy)), } err := binary.Write(w, binary.LittleEndian, header) if err != nil { return 0, err } n := binary.Size(header) m, err := io.WriteString(w, string(proxy)) if err != nil { return 0, err } return n + m, nil } func (p *Pickler) dumpString(v string) { p.pushProxy(stringProxy(v)) } stalecucumber-master/LICENSE0000644000000000000000000000241013011632561014703 0ustar rootrootCopyright (c) 2014, Eric Urban All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.