From 16372365c47cc9387d40325ea51ecef63e949c75 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 21 Oct 2015 12:15:19 -0700 Subject: [PATCH] First commit --- array.go | 5 + async.go | 15 +++ bit_array.go | 275 ++++++++++++++++++++++++++++++++++++++++++++++ bit_array_test.go | 120 ++++++++++++++++++++ byteslice.go | 44 ++++++++ cmap.go | 62 +++++++++++ colors.go | 84 ++++++++++++++ errors.go | 45 ++++++++ heap.go | 103 +++++++++++++++++ int.go | 55 ++++++++++ io.go | 75 +++++++++++++ math.go | 157 ++++++++++++++++++++++++++ os.go | 225 +++++++++++++++++++++++++++++++++++++ random.go | 145 ++++++++++++++++++++++++ repeat_timer.go | 72 ++++++++++++ service.go | 154 ++++++++++++++++++++++++++ string.go | 24 ++++ test/assert.go | 14 +++ test/mutate.go | 28 +++++ throttle_timer.go | 57 ++++++++++ word.go | 91 +++++++++++++++ 21 files changed, 1850 insertions(+) create mode 100644 array.go create mode 100644 async.go create mode 100644 bit_array.go create mode 100644 bit_array_test.go create mode 100644 byteslice.go create mode 100644 cmap.go create mode 100644 colors.go create mode 100644 errors.go create mode 100644 heap.go create mode 100644 int.go create mode 100644 io.go create mode 100644 math.go create mode 100644 os.go create mode 100644 random.go create mode 100644 repeat_timer.go create mode 100644 service.go create mode 100644 string.go create mode 100644 test/assert.go create mode 100644 test/mutate.go create mode 100644 throttle_timer.go create mode 100644 word.go diff --git a/array.go b/array.go new file mode 100644 index 000000000..adedc42be --- /dev/null +++ b/array.go @@ -0,0 +1,5 @@ +package common + +func Arr(items ...interface{}) []interface{} { + return items +} diff --git a/async.go b/async.go new file mode 100644 index 000000000..1d302c344 --- /dev/null +++ b/async.go @@ -0,0 +1,15 @@ +package common + +import "sync" + +func Parallel(tasks ...func()) { + var wg sync.WaitGroup + wg.Add(len(tasks)) + for _, task := range tasks { + go func(task func()) { + task() + wg.Done() + }(task) + } + wg.Wait() +} diff --git a/bit_array.go b/bit_array.go new file mode 100644 index 000000000..dc006f0eb --- /dev/null +++ b/bit_array.go @@ -0,0 +1,275 @@ +package common + +import ( + "fmt" + "math/rand" + "strings" + "sync" +) + +type BitArray struct { + mtx sync.Mutex + Bits int `json:"bits"` // NOTE: persisted via reflect, must be exported + Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported +} + +// There is no BitArray whose Size is 0. Use nil instead. +func NewBitArray(bits int) *BitArray { + if bits == 0 { + return nil + } + return &BitArray{ + Bits: bits, + Elems: make([]uint64, (bits+63)/64), + } +} + +func (bA *BitArray) Size() int { + if bA == nil { + return 0 + } + return bA.Bits +} + +// NOTE: behavior is undefined if i >= bA.Bits +func (bA *BitArray) GetIndex(i int) bool { + if bA == nil { + return false + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.getIndex(i) +} + +func (bA *BitArray) getIndex(i int) bool { + if i >= bA.Bits { + return false + } + return bA.Elems[i/64]&(uint64(1)< 0 +} + +// NOTE: behavior is undefined if i >= bA.Bits +func (bA *BitArray) SetIndex(i int, v bool) bool { + if bA == nil { + return false + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.setIndex(i, v) +} + +func (bA *BitArray) setIndex(i int, v bool) bool { + if i >= bA.Bits { + return false + } + if v { + bA.Elems[i/64] |= (uint64(1) << uint(i%64)) + } else { + bA.Elems[i/64] &= ^(uint64(1) << uint(i%64)) + } + return true +} + +func (bA *BitArray) Copy() *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.copy() +} + +func (bA *BitArray) copy() *BitArray { + c := make([]uint64, len(bA.Elems)) + copy(c, bA.Elems) + return &BitArray{ + Bits: bA.Bits, + Elems: c, + } +} + +func (bA *BitArray) copyBits(bits int) *BitArray { + c := make([]uint64, (bits+63)/64) + copy(c, bA.Elems) + return &BitArray{ + Bits: bits, + Elems: c, + } +} + +// Returns a BitArray of larger bits size. +func (bA *BitArray) Or(o *BitArray) *BitArray { + if bA == nil { + o.Copy() + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + c := bA.copyBits(MaxInt(bA.Bits, o.Bits)) + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] |= o.Elems[i] + } + return c +} + +// Returns a BitArray of smaller bit size. +func (bA *BitArray) And(o *BitArray) *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.and(o) +} + +func (bA *BitArray) and(o *BitArray) *BitArray { + c := bA.copyBits(MinInt(bA.Bits, o.Bits)) + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] &= o.Elems[i] + } + return c +} + +func (bA *BitArray) Not() *BitArray { + if bA == nil { + return nil // Degenerate + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + c := bA.copy() + for i := 0; i < len(c.Elems); i++ { + c.Elems[i] = ^c.Elems[i] + } + return c +} + +func (bA *BitArray) Sub(o *BitArray) *BitArray { + if bA == nil { + return nil + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + if bA.Bits > o.Bits { + c := bA.copy() + for i := 0; i < len(o.Elems)-1; i++ { + c.Elems[i] &= ^c.Elems[i] + } + i := len(o.Elems) - 1 + if i >= 0 { + for idx := i * 64; idx < o.Bits; idx++ { + // NOTE: each individual GetIndex() call to o is safe. + c.setIndex(idx, c.getIndex(idx) && !o.GetIndex(idx)) + } + } + return c + } else { + return bA.and(o.Not()) // Note degenerate case where o == nil + } +} + +func (bA *BitArray) IsFull() bool { + if bA == nil { + return true + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + + // Check all elements except the last + for _, elem := range bA.Elems[:len(bA.Elems)-1] { + if (^elem) != 0 { + return false + } + } + + // Check that the last element has (lastElemBits) 1's + lastElemBits := (bA.Bits+63)%64 + 1 + lastElem := bA.Elems[len(bA.Elems)-1] + return (lastElem+1)&((uint64(1)< 0 { + randBitStart := rand.Intn(64) + for j := 0; j < 64; j++ { + bitIdx := ((j + randBitStart) % 64) + if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 { + return 64*elemIdx + bitIdx, true + } + } + PanicSanity("should not happen") + } + } else { + // Special case for last elem, to ignore straggler bits + elemBits := bA.Bits % 64 + if elemBits == 0 { + elemBits = 64 + } + randBitStart := rand.Intn(elemBits) + for j := 0; j < elemBits; j++ { + bitIdx := ((j + randBitStart) % elemBits) + if (bA.Elems[elemIdx] & (uint64(1) << uint(bitIdx))) > 0 { + return 64*elemIdx + bitIdx, true + } + } + } + } + return 0, false +} + +func (bA *BitArray) String() string { + if bA == nil { + return "nil-BitArray" + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.stringIndented("") +} + +func (bA *BitArray) StringIndented(indent string) string { + if bA == nil { + return "nil-BitArray" + } + bA.mtx.Lock() + defer bA.mtx.Unlock() + return bA.stringIndented(indent) +} + +func (bA *BitArray) stringIndented(indent string) string { + + lines := []string{} + bits := "" + for i := 0; i < bA.Bits; i++ { + if bA.getIndex(i) { + bits += "X" + } else { + bits += "_" + } + if i%100 == 99 { + lines = append(lines, bits) + bits = "" + } + if i%10 == 9 { + bits += " " + } + if i%50 == 49 { + bits += " " + } + } + if len(bits) > 0 { + lines = append(lines, bits) + } + return fmt.Sprintf("BA{%v:%v}", bA.Bits, strings.Join(lines, indent)) +} diff --git a/bit_array_test.go b/bit_array_test.go new file mode 100644 index 000000000..93274aab0 --- /dev/null +++ b/bit_array_test.go @@ -0,0 +1,120 @@ +package common + +import ( + "testing" +) + +func randBitArray(bits int) (*BitArray, []byte) { + src := RandBytes((bits + 7) / 8) + bA := NewBitArray(bits) + for i := 0; i < len(src); i++ { + for j := 0; j < 8; j++ { + if i*8+j >= bits { + return bA, src + } + setBit := src[i]&(1< 0 + bA.SetIndex(i*8+j, setBit) + } + } + return bA, src +} + +func TestAnd(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.And(bA2) + + if bA3.Bits != 31 { + t.Error("Expected min bits", bA3.Bits) + } + if len(bA3.Elems) != len(bA2.Elems) { + t.Error("Expected min elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) && bA2.GetIndex(i) + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestOr(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.Or(bA2) + + if bA3.Bits != 51 { + t.Error("Expected max bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected max elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) || bA2.GetIndex(i) + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestSub1(t *testing.T) { + + bA1, _ := randBitArray(31) + bA2, _ := randBitArray(51) + bA3 := bA1.Sub(bA2) + + if bA3.Bits != bA1.Bits { + t.Error("Expected bA1 bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected bA1 elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) + if bA2.GetIndex(i) { + expected = false + } + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3", i, bA1.GetIndex(i), bA2.GetIndex(i), bA3.GetIndex(i)) + } + } +} + +func TestSub2(t *testing.T) { + + bA1, _ := randBitArray(51) + bA2, _ := randBitArray(31) + bA3 := bA1.Sub(bA2) + + if bA3.Bits != bA1.Bits { + t.Error("Expected bA1 bits") + } + if len(bA3.Elems) != len(bA1.Elems) { + t.Error("Expected bA1 elems length") + } + for i := 0; i < bA3.Bits; i++ { + expected := bA1.GetIndex(i) + if i < bA2.Bits && bA2.GetIndex(i) { + expected = false + } + if bA3.GetIndex(i) != expected { + t.Error("Wrong bit from bA3") + } + } +} + +func TestPickRandom(t *testing.T) { + for idx := 0; idx < 123; idx++ { + bA1 := NewBitArray(123) + bA1.SetIndex(idx, true) + index, ok := bA1.PickRandom() + if !ok { + t.Fatal("Expected to pick element but got none") + } + if index != idx { + t.Fatalf("Expected to pick element at %v but got wrong index", idx) + } + } +} diff --git a/byteslice.go b/byteslice.go new file mode 100644 index 000000000..be828f065 --- /dev/null +++ b/byteslice.go @@ -0,0 +1,44 @@ +package common + +import ( + "bytes" +) + +func Fingerprint(slice []byte) []byte { + fingerprint := make([]byte, 6) + copy(fingerprint, slice) + return fingerprint +} + +func IsZeros(slice []byte) bool { + for _, byt := range slice { + if byt != byte(0) { + return false + } + } + return true +} + +func RightPadBytes(slice []byte, l int) []byte { + if l < len(slice) { + return slice + } + padded := make([]byte, l) + copy(padded[0:len(slice)], slice) + return padded +} + +func LeftPadBytes(slice []byte, l int) []byte { + if l < len(slice) { + return slice + } + padded := make([]byte, l) + copy(padded[l-len(slice):], slice) + return padded +} + +func TrimmedString(b []byte) string { + trimSet := string([]byte{0}) + return string(bytes.TrimLeft(b, trimSet)) + +} diff --git a/cmap.go b/cmap.go new file mode 100644 index 000000000..5de6fa2fa --- /dev/null +++ b/cmap.go @@ -0,0 +1,62 @@ +package common + +import "sync" + +// CMap is a goroutine-safe map +type CMap struct { + m map[string]interface{} + l sync.Mutex +} + +func NewCMap() *CMap { + return &CMap{ + m: make(map[string]interface{}, 0), + } +} + +func (cm *CMap) Set(key string, value interface{}) { + cm.l.Lock() + defer cm.l.Unlock() + cm.m[key] = value +} + +func (cm *CMap) Get(key string) interface{} { + cm.l.Lock() + defer cm.l.Unlock() + return cm.m[key] +} + +func (cm *CMap) Has(key string) bool { + cm.l.Lock() + defer cm.l.Unlock() + _, ok := cm.m[key] + return ok +} + +func (cm *CMap) Delete(key string) { + cm.l.Lock() + defer cm.l.Unlock() + delete(cm.m, key) +} + +func (cm *CMap) Size() int { + cm.l.Lock() + defer cm.l.Unlock() + return len(cm.m) +} + +func (cm *CMap) Clear() { + cm.l.Lock() + defer cm.l.Unlock() + cm.m = make(map[string]interface{}, 0) +} + +func (cm *CMap) Values() []interface{} { + cm.l.Lock() + defer cm.l.Unlock() + items := []interface{}{} + for _, v := range cm.m { + items = append(items, v) + } + return items +} diff --git a/colors.go b/colors.go new file mode 100644 index 000000000..776b22e2e --- /dev/null +++ b/colors.go @@ -0,0 +1,84 @@ +package common + +import ( + "fmt" + "strings" +) + +const ( + ANSIReset = "\x1b[0m" + ANSIBright = "\x1b[1m" + ANSIDim = "\x1b[2m" + ANSIUnderscore = "\x1b[4m" + ANSIBlink = "\x1b[5m" + ANSIReverse = "\x1b[7m" + ANSIHidden = "\x1b[8m" + + ANSIFgBlack = "\x1b[30m" + ANSIFgRed = "\x1b[31m" + ANSIFgGreen = "\x1b[32m" + ANSIFgYellow = "\x1b[33m" + ANSIFgBlue = "\x1b[34m" + ANSIFgMagenta = "\x1b[35m" + ANSIFgCyan = "\x1b[36m" + ANSIFgWhite = "\x1b[37m" + + ANSIBgBlack = "\x1b[40m" + ANSIBgRed = "\x1b[41m" + ANSIBgGreen = "\x1b[42m" + ANSIBgYellow = "\x1b[43m" + ANSIBgBlue = "\x1b[44m" + ANSIBgMagenta = "\x1b[45m" + ANSIBgCyan = "\x1b[46m" + ANSIBgWhite = "\x1b[47m" +) + +// color the string s with color 'color' +// unless s is already colored +func treat(s string, color string) string { + if len(s) > 2 && s[:2] == "\x1b[" { + return s + } else { + return color + s + ANSIReset + } +} + +func treatAll(color string, args ...interface{}) string { + var parts []string + for _, arg := range args { + parts = append(parts, treat(fmt.Sprintf("%v", arg), color)) + } + return strings.Join(parts, "") +} + +func Black(args ...interface{}) string { + return treatAll(ANSIFgBlack, args...) +} + +func Red(args ...interface{}) string { + return treatAll(ANSIFgRed, args...) +} + +func Green(args ...interface{}) string { + return treatAll(ANSIFgGreen, args...) +} + +func Yellow(args ...interface{}) string { + return treatAll(ANSIFgYellow, args...) +} + +func Blue(args ...interface{}) string { + return treatAll(ANSIFgBlue, args...) +} + +func Magenta(args ...interface{}) string { + return treatAll(ANSIFgMagenta, args...) +} + +func Cyan(args ...interface{}) string { + return treatAll(ANSIFgCyan, args...) +} + +func White(args ...interface{}) string { + return treatAll(ANSIFgWhite, args...) +} diff --git a/errors.go b/errors.go new file mode 100644 index 000000000..e168a75b7 --- /dev/null +++ b/errors.go @@ -0,0 +1,45 @@ +package common + +import ( + "fmt" +) + +type StackError struct { + Err interface{} + Stack []byte +} + +func (se StackError) String() string { + return fmt.Sprintf("Error: %v\nStack: %s", se.Err, se.Stack) +} + +func (se StackError) Error() string { + return se.String() +} + +//-------------------------------------------------------------------------------------------------- +// panic wrappers + +// A panic resulting from a sanity check means there is a programmer error +// and some gaurantee is not satisfied. +func PanicSanity(v interface{}) { + panic(Fmt("Paniced on a Sanity Check: %v", v)) +} + +// A panic here means something has gone horribly wrong, in the form of data corruption or +// failure of the operating system. In a correct/healthy system, these should never fire. +// If they do, it's indicative of a much more serious problem. +func PanicCrisis(v interface{}) { + panic(Fmt("Paniced on a Crisis: %v", v)) +} + +// Indicates a failure of consensus. Someone was malicious or something has +// gone horribly wrong. These should really boot us into an "emergency-recover" mode +func PanicConsensus(v interface{}) { + panic(Fmt("Paniced on a Consensus Failure: %v", v)) +} + +// For those times when we're not sure if we should panic +func PanicQ(v interface{}) { + panic(Fmt("Paniced questionably: %v", v)) +} diff --git a/heap.go b/heap.go new file mode 100644 index 000000000..4a96d7aaa --- /dev/null +++ b/heap.go @@ -0,0 +1,103 @@ +package common + +import ( + "container/heap" +) + +type Comparable interface { + Less(o interface{}) bool +} + +//----------------------------------------------------------------------------- + +/* +Example usage: + h := NewHeap() + + h.Push(String("msg1"), 1) + h.Push(String("msg3"), 3) + h.Push(String("msg2"), 2) + + fmt.Println(h.Pop()) + fmt.Println(h.Pop()) + fmt.Println(h.Pop()) +*/ + +type Heap struct { + pq priorityQueue +} + +func NewHeap() *Heap { + return &Heap{pq: make([]*pqItem, 0)} +} + +func (h *Heap) Len() int64 { + return int64(len(h.pq)) +} + +func (h *Heap) Push(value interface{}, priority Comparable) { + heap.Push(&h.pq, &pqItem{value: value, priority: priority}) +} + +func (h *Heap) Peek() interface{} { + if len(h.pq) == 0 { + return nil + } + return h.pq[0].value +} + +func (h *Heap) Update(value interface{}, priority Comparable) { + h.pq.Update(h.pq[0], value, priority) +} + +func (h *Heap) Pop() interface{} { + item := heap.Pop(&h.pq).(*pqItem) + return item.value +} + +//----------------------------------------------------------------------------- + +/////////////////////// +// From: http://golang.org/pkg/container/heap/#example__priorityQueue + +type pqItem struct { + value interface{} + priority Comparable + index int +} + +type priorityQueue []*pqItem + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].priority.Less(pq[j].priority) +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*pqItem) + item.index = n + *pq = append(*pq, item) +} + +func (pq *priorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +func (pq *priorityQueue) Update(item *pqItem, value interface{}, priority Comparable) { + item.value = value + item.priority = priority + heap.Fix(pq, item.index) +} diff --git a/int.go b/int.go new file mode 100644 index 000000000..50e86a072 --- /dev/null +++ b/int.go @@ -0,0 +1,55 @@ +package common + +import ( + "encoding/binary" + "sort" +) + +// Sort for []uint64 + +type Uint64Slice []uint64 + +func (p Uint64Slice) Len() int { return len(p) } +func (p Uint64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p Uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Uint64Slice) Sort() { sort.Sort(p) } + +func SearchUint64s(a []uint64, x uint64) int { + return sort.Search(len(a), func(i int) bool { return a[i] >= x }) +} + +func (p Uint64Slice) Search(x uint64) int { return SearchUint64s(p, x) } + +//----------------------------------------------------------------------------- + +func PutUint64LE(dest []byte, i uint64) { + binary.LittleEndian.PutUint64(dest, i) +} + +func GetUint64LE(src []byte) uint64 { + return binary.LittleEndian.Uint64(src) +} + +func PutUint64BE(dest []byte, i uint64) { + binary.BigEndian.PutUint64(dest, i) +} + +func GetUint64BE(src []byte) uint64 { + return binary.BigEndian.Uint64(src) +} + +func PutInt64LE(dest []byte, i int64) { + binary.LittleEndian.PutUint64(dest, uint64(i)) +} + +func GetInt64LE(src []byte) int64 { + return int64(binary.LittleEndian.Uint64(src)) +} + +func PutInt64BE(dest []byte, i int64) { + binary.BigEndian.PutUint64(dest, uint64(i)) +} + +func GetInt64BE(src []byte) int64 { + return int64(binary.BigEndian.Uint64(src)) +} diff --git a/io.go b/io.go new file mode 100644 index 000000000..378c19fc6 --- /dev/null +++ b/io.go @@ -0,0 +1,75 @@ +package common + +import ( + "bytes" + "errors" + "io" +) + +type PrefixedReader struct { + Prefix []byte + reader io.Reader +} + +func NewPrefixedReader(prefix []byte, reader io.Reader) *PrefixedReader { + return &PrefixedReader{prefix, reader} +} + +func (pr *PrefixedReader) Read(p []byte) (n int, err error) { + if len(pr.Prefix) > 0 { + read := copy(p, pr.Prefix) + pr.Prefix = pr.Prefix[read:] + return read, nil + } else { + return pr.reader.Read(p) + } +} + +// NOTE: Not goroutine safe +type BufferCloser struct { + bytes.Buffer + Closed bool +} + +func NewBufferCloser(buf []byte) *BufferCloser { + return &BufferCloser{ + *bytes.NewBuffer(buf), + false, + } +} + +func (bc *BufferCloser) Close() error { + if bc.Closed { + return errors.New("BufferCloser already closed") + } + bc.Closed = true + return nil +} + +func (bc *BufferCloser) Write(p []byte) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.Write(p) +} + +func (bc *BufferCloser) WriteByte(c byte) error { + if bc.Closed { + return errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteByte(c) +} + +func (bc *BufferCloser) WriteRune(r rune) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteRune(r) +} + +func (bc *BufferCloser) WriteString(s string) (n int, err error) { + if bc.Closed { + return 0, errors.New("Cannot write to closed BufferCloser") + } + return bc.Buffer.WriteString(s) +} diff --git a/math.go b/math.go new file mode 100644 index 000000000..b037d1a71 --- /dev/null +++ b/math.go @@ -0,0 +1,157 @@ +package common + +func MaxInt8(a, b int8) int8 { + if a > b { + return a + } + return b +} + +func MaxUint8(a, b uint8) uint8 { + if a > b { + return a + } + return b +} + +func MaxInt16(a, b int16) int16 { + if a > b { + return a + } + return b +} + +func MaxUint16(a, b uint16) uint16 { + if a > b { + return a + } + return b +} + +func MaxInt32(a, b int32) int32 { + if a > b { + return a + } + return b +} + +func MaxUint32(a, b uint32) uint32 { + if a > b { + return a + } + return b +} + +func MaxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func MaxUint64(a, b uint64) uint64 { + if a > b { + return a + } + return b +} + +func MaxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func MaxUint(a, b uint) uint { + if a > b { + return a + } + return b +} + +//----------------------------------------------------------------------------- + +func MinInt8(a, b int8) int8 { + if a < b { + return a + } + return b +} + +func MinUint8(a, b uint8) uint8 { + if a < b { + return a + } + return b +} + +func MinInt16(a, b int16) int16 { + if a < b { + return a + } + return b +} + +func MinUint16(a, b uint16) uint16 { + if a < b { + return a + } + return b +} + +func MinInt32(a, b int32) int32 { + if a < b { + return a + } + return b +} + +func MinUint32(a, b uint32) uint32 { + if a < b { + return a + } + return b +} + +func MinInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +func MinUint64(a, b uint64) uint64 { + if a < b { + return a + } + return b +} + +func MinInt(a, b int) int { + if a < b { + return a + } + return b +} + +func MinUint(a, b uint) uint { + if a < b { + return a + } + return b +} + +//----------------------------------------------------------------------------- + +func ExpUint64(a, b uint64) uint64 { + accum := uint64(1) + for b > 0 { + if b&1 == 1 { + accum *= a + } + a *= a + b >>= 1 + } + return accum +} diff --git a/os.go b/os.go new file mode 100644 index 000000000..170c6f82a --- /dev/null +++ b/os.go @@ -0,0 +1,225 @@ +package common + +import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "os/signal" + "strings" + "sync" + "time" +) + +var ( + GoPath = os.Getenv("GOPATH") +) + +func TrapSignal(cb func()) { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + signal.Notify(c, os.Kill) + go func() { + for sig := range c { + fmt.Printf("captured %v, exiting...\n", sig) + if cb != nil { + cb() + } + os.Exit(1) + } + }() + select {} +} + +func Exit(s string) { + fmt.Printf(s + "\n") + os.Exit(1) +} + +func EnsureDir(dir string) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + err := os.MkdirAll(dir, 0700) + if err != nil { + return fmt.Errorf("Could not create directory %v. %v", dir, err) + } + } + return nil +} + +func FileExists(filePath string) bool { + _, err := os.Stat(filePath) + return !os.IsNotExist(err) +} + +func ReadFile(filePath string) ([]byte, error) { + return ioutil.ReadFile(filePath) +} + +func MustReadFile(filePath string) []byte { + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + Exit(Fmt("MustReadFile failed: %v", err)) + return nil + } + return fileBytes +} + +func WriteFile(filePath string, contents []byte) error { + err := ioutil.WriteFile(filePath, contents, 0600) + if err != nil { + return err + } + // fmt.Printf("File written to %v.\n", filePath) + return nil +} + +func MustWriteFile(filePath string, contents []byte) { + err := WriteFile(filePath, contents) + if err != nil { + Exit(Fmt("MustWriteFile failed: %v", err)) + } +} + +// Writes to newBytes to filePath. +// Guaranteed not to lose *both* oldBytes and newBytes, +// (assuming that the OS is perfect) +func WriteFileAtomic(filePath string, newBytes []byte) error { + // If a file already exists there, copy to filePath+".bak" (overwrite anything) + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + return fmt.Errorf("Could not read file %v. %v", filePath, err) + } + err = ioutil.WriteFile(filePath+".bak", fileBytes, 0600) + if err != nil { + return fmt.Errorf("Could not write file %v. %v", filePath+".bak", err) + } + } + // Write newBytes to filePath.new + err := ioutil.WriteFile(filePath+".new", newBytes, 0600) + if err != nil { + return fmt.Errorf("Could not write file %v. %v", filePath+".new", err) + } + // Move filePath.new to filePath + err = os.Rename(filePath+".new", filePath) + return err +} + +//-------------------------------------------------------------------------------- + +/* AutoFile usage + +// Create/Append to ./autofile_test +af, err := OpenAutoFile("autofile_test") +if err != nil { + panic(err) +} + +// Stream of writes. +// During this time, the file may be moved e.g. by logRotate. +for i := 0; i < 60; i++ { + af.Write([]byte(Fmt("LOOP(%v)", i))) + time.Sleep(time.Second) +} + +// Close the AutoFile +err = af.Close() +if err != nil { + panic(err) +} +*/ + +const autoFileOpenDuration = 1000 * time.Millisecond + +// Automatically closes and re-opens file for writing. +// This is useful for using a log file with the logrotate tool. +type AutoFile struct { + Path string + ticker *time.Ticker + mtx sync.Mutex + file *os.File +} + +func OpenAutoFile(path string) (af *AutoFile, err error) { + af = &AutoFile{ + Path: path, + ticker: time.NewTicker(autoFileOpenDuration), + } + if err = af.openFile(); err != nil { + return + } + go af.processTicks() + return +} + +func (af *AutoFile) Close() error { + af.ticker.Stop() + af.mtx.Lock() + err := af.closeFile() + af.mtx.Unlock() + return err +} + +func (af *AutoFile) processTicks() { + for { + _, ok := <-af.ticker.C + if !ok { + return // Done. + } + af.mtx.Lock() + af.closeFile() + af.mtx.Unlock() + } +} + +func (af *AutoFile) closeFile() (err error) { + file := af.file + if file == nil { + return nil + } + af.file = nil + return file.Close() +} + +func (af *AutoFile) Write(b []byte) (n int, err error) { + af.mtx.Lock() + defer af.mtx.Unlock() + if af.file == nil { + if err = af.openFile(); err != nil { + return + } + } + return af.file.Write(b) +} + +func (af *AutoFile) openFile() error { + file, err := os.OpenFile(af.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return err + } + af.file = file + return nil +} + +func Tempfile(prefix string) (*os.File, string) { + file, err := ioutil.TempFile("", prefix) + if err != nil { + PanicCrisis(err) + } + return file, file.Name() +} + +func Prompt(prompt string, defaultValue string) (string, error) { + fmt.Print(prompt) + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + return defaultValue, err + } else { + line = strings.TrimSpace(line) + if line == "" { + return defaultValue, nil + } + return line, nil + } +} diff --git a/random.go b/random.go new file mode 100644 index 000000000..645601154 --- /dev/null +++ b/random.go @@ -0,0 +1,145 @@ +package common + +import ( + crand "crypto/rand" + "encoding/hex" + "math/rand" + "time" +) + +const ( + strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters +) + +func init() { + // Seed math/rand with "secure" int64 + b := CRandBytes(8) + var seed uint64 + for i := 0; i < 8; i++ { + seed |= uint64(b[i]) + seed <<= 8 + } + rand.Seed(int64(seed)) +} + +// Constructs an alphanumeric string of given length. +func RandStr(length int) string { + chars := []byte{} +MAIN_LOOP: + for { + val := rand.Int63() + for i := 0; i < 10; i++ { + v := int(val & 0x3f) // rightmost 6 bits + if v >= 62 { // only 62 characters in strChars + val >>= 6 + continue + } else { + chars = append(chars, strChars[v]) + if len(chars) == length { + break MAIN_LOOP + } + val >>= 6 + } + } + } + + return string(chars) +} + +func RandUint16() uint16 { + return uint16(rand.Uint32() & (1<<16 - 1)) +} + +func RandUint32() uint32 { + return rand.Uint32() +} + +func RandUint64() uint64 { + return uint64(rand.Uint32())<<32 + uint64(rand.Uint32()) +} + +func RandUint() uint { + return uint(rand.Int()) +} + +func RandInt16() int16 { + return int16(rand.Uint32() & (1<<16 - 1)) +} + +func RandInt32() int32 { + return int32(rand.Uint32()) +} + +func RandInt64() int64 { + return int64(rand.Uint32())<<32 + int64(rand.Uint32()) +} + +func RandInt() int { + return rand.Int() +} + +// Distributed pseudo-exponentially to test for various cases +func RandUint16Exp() uint16 { + bits := rand.Uint32() % 16 + if bits == 0 { + return 0 + } + n := uint16(1 << (bits - 1)) + n += uint16(rand.Int31()) & ((1 << (bits - 1)) - 1) + return n +} + +// Distributed pseudo-exponentially to test for various cases +func RandUint32Exp() uint32 { + bits := rand.Uint32() % 32 + if bits == 0 { + return 0 + } + n := uint32(1 << (bits - 1)) + n += uint32(rand.Int31()) & ((1 << (bits - 1)) - 1) + return n +} + +// Distributed pseudo-exponentially to test for various cases +func RandUint64Exp() uint64 { + bits := rand.Uint32() % 64 + if bits == 0 { + return 0 + } + n := uint64(1 << (bits - 1)) + n += uint64(rand.Int63()) & ((1 << (bits - 1)) - 1) + return n +} + +func RandFloat32() float32 { + return rand.Float32() +} + +func RandTime() time.Time { + return time.Unix(int64(RandUint64Exp()), 0) +} + +func RandBytes(n int) []byte { + bs := make([]byte, n) + for i := 0; i < n; i++ { + bs[i] = byte(rand.Intn(256)) + } + return bs +} + +//----------------------------------------------------------------------------- +// CRand* methods are crypto safe. + +func CRandBytes(numBytes int) []byte { + b := make([]byte, numBytes) + _, err := crand.Read(b) + if err != nil { + PanicCrisis(err) + } + return b +} + +// RandHex(24) gives 96 bits of randomness, strong enough for most purposes. +func CRandHex(numDigits int) string { + return hex.EncodeToString(CRandBytes(numDigits / 2)) +} diff --git a/repeat_timer.go b/repeat_timer.go new file mode 100644 index 000000000..e2aa18ea8 --- /dev/null +++ b/repeat_timer.go @@ -0,0 +1,72 @@ +package common + +import "time" +import "sync" + +/* +RepeatTimer repeatedly sends a struct{}{} to .Ch after each "dur" period. +It's good for keeping connections alive. +A RepeatTimer must be Stop()'d or it will keep a goroutine alive. +*/ +type RepeatTimer struct { + Ch chan time.Time + + mtx sync.Mutex + name string + ticker *time.Ticker + quit chan struct{} + dur time.Duration +} + +func NewRepeatTimer(name string, dur time.Duration) *RepeatTimer { + var t = &RepeatTimer{ + Ch: make(chan time.Time), + ticker: time.NewTicker(dur), + quit: make(chan struct{}), + name: name, + dur: dur, + } + go t.fireRoutine(t.ticker) + return t +} + +func (t *RepeatTimer) fireRoutine(ticker *time.Ticker) { + for { + select { + case t_ := <-ticker.C: + t.Ch <- t_ + case <-t.quit: + return + } + } +} + +// Wait the duration again before firing. +func (t *RepeatTimer) Reset() { + t.Stop() + + t.mtx.Lock() // Lock + defer t.mtx.Unlock() + + t.ticker = time.NewTicker(t.dur) + t.quit = make(chan struct{}) + go t.fireRoutine(t.ticker) +} + +// For ease of .Stop()'ing services before .Start()'ing them, +// we ignore .Stop()'s on nil RepeatTimers. +func (t *RepeatTimer) Stop() bool { + if t == nil { + return false + } + t.mtx.Lock() // Lock + defer t.mtx.Unlock() + + exists := t.ticker != nil + if exists { + t.ticker.Stop() + t.ticker = nil + close(t.quit) + } + return exists +} diff --git a/service.go b/service.go new file mode 100644 index 000000000..05b2adedd --- /dev/null +++ b/service.go @@ -0,0 +1,154 @@ +/* + +Classical-inheritance-style service declarations. +Services can be started, then stopped. +Users can override the OnStart/OnStop methods. +These methods are guaranteed to be called at most once. +Caller must ensure that Start() and Stop() are not called concurrently. +It is ok to call Stop() without calling Start() first. +Services cannot be re-started unless otherwise documented. + +Typical usage: + +type FooService struct { + BaseService + // private fields +} + +func NewFooService() *FooService { + fs := &FooService{ + // init + } + fs.BaseService = *NewBaseService(log, "FooService", fs) + return fs +} + +func (fs *FooService) OnStart() error { + fs.BaseService.OnStart() // Always call the overridden method. + // initialize private fields + // start subroutines, etc. +} + +func (fs *FooService) OnStop() error { + fs.BaseService.OnStop() // Always call the overridden method. + // close/destroy private fields + // stop subroutines, etc. +} + +*/ +package common + +import "sync/atomic" +import "github.com/tendermint/tendermint/Godeps/_workspace/src/github.com/tendermint/log15" + +type Service interface { + Start() (bool, error) + OnStart() error + + Stop() bool + OnStop() + + IsRunning() bool + + String() string +} + +type BaseService struct { + log log15.Logger + name string + started uint32 // atomic + stopped uint32 // atomic + + // The "subclass" of BaseService + impl Service +} + +func NewBaseService(log log15.Logger, name string, impl Service) *BaseService { + return &BaseService{ + log: log, + name: name, + impl: impl, + } +} + +// Implements Servce +func (bs *BaseService) Start() (bool, error) { + if atomic.CompareAndSwapUint32(&bs.started, 0, 1) { + if atomic.LoadUint32(&bs.stopped) == 1 { + if bs.log != nil { + bs.log.Warn(Fmt("Not starting %v -- already stopped", bs.name), "impl", bs.impl) + } + return false, nil + } else { + if bs.log != nil { + bs.log.Notice(Fmt("Starting %v", bs.name), "impl", bs.impl) + } + } + err := bs.impl.OnStart() + return true, err + } else { + if bs.log != nil { + bs.log.Info(Fmt("Not starting %v -- already started", bs.name), "impl", bs.impl) + } + return false, nil + } +} + +// Implements Service +func (bs *BaseService) OnStart() error { return nil } + +// Implements Service +func (bs *BaseService) Stop() bool { + if atomic.CompareAndSwapUint32(&bs.stopped, 0, 1) { + if bs.log != nil { + bs.log.Notice(Fmt("Stopping %v", bs.name), "impl", bs.impl) + } + bs.impl.OnStop() + return true + } else { + if bs.log != nil { + bs.log.Notice(Fmt("Not stopping %v", bs.name), "impl", bs.impl) + } + return false + } +} + +// Implements Service +func (bs *BaseService) OnStop() {} + +// Implements Service +func (bs *BaseService) IsRunning() bool { + return atomic.LoadUint32(&bs.started) == 1 && atomic.LoadUint32(&bs.stopped) == 0 +} + +// Implements Servce +func (bs *BaseService) String() string { + return bs.name +} + +//---------------------------------------- + +type QuitService struct { + BaseService + Quit chan struct{} +} + +func NewQuitService(log log15.Logger, name string, impl Service) *QuitService { + return &QuitService{ + BaseService: *NewBaseService(log, name, impl), + Quit: nil, + } +} + +// NOTE: when overriding OnStart, must call .QuitService.OnStart(). +func (qs *QuitService) OnStart() error { + qs.Quit = make(chan struct{}) + return nil +} + +// NOTE: when overriding OnStop, must call .QuitService.OnStop(). +func (qs *QuitService) OnStop() { + if qs.Quit != nil { + close(qs.Quit) + } +} diff --git a/string.go b/string.go new file mode 100644 index 000000000..a4d221b74 --- /dev/null +++ b/string.go @@ -0,0 +1,24 @@ +package common + +import ( + "fmt" + "strings" +) + +var Fmt = fmt.Sprintf + +func RightPadString(s string, totalLength int) string { + remaining := totalLength - len(s) + if remaining > 0 { + s = s + strings.Repeat(" ", remaining) + } + return s +} + +func LeftPadString(s string, totalLength int) string { + remaining := totalLength - len(s) + if remaining > 0 { + s = strings.Repeat(" ", remaining) + s + } + return s +} diff --git a/test/assert.go b/test/assert.go new file mode 100644 index 000000000..a6ffed0ce --- /dev/null +++ b/test/assert.go @@ -0,0 +1,14 @@ +package test + +import ( + "testing" +) + +func AssertPanics(t *testing.T, msg string, f func()) { + defer func() { + if err := recover(); err == nil { + t.Errorf("Should have panic'd, but didn't: %v", msg) + } + }() + f() +} diff --git a/test/mutate.go b/test/mutate.go new file mode 100644 index 000000000..39bf90557 --- /dev/null +++ b/test/mutate.go @@ -0,0 +1,28 @@ +package test + +import ( + . "github.com/tendermint/tendermint/common" +) + +// Contract: !bytes.Equal(input, output) && len(input) >= len(output) +func MutateByteSlice(bytez []byte) []byte { + // If bytez is empty, panic + if len(bytez) == 0 { + panic("Cannot mutate an empty bytez") + } + + // Copy bytez + mBytez := make([]byte, len(bytez)) + copy(mBytez, bytez) + bytez = mBytez + + // Try a random mutation + switch RandInt() % 2 { + case 0: // Mutate a single byte + bytez[RandInt()%len(bytez)] += byte(RandInt()%255 + 1) + case 1: // Remove an arbitrary byte + pos := RandInt() % len(bytez) + bytez = append(bytez[:pos], bytez[pos+1:]...) + } + return bytez +} diff --git a/throttle_timer.go b/throttle_timer.go new file mode 100644 index 000000000..0b40a60c2 --- /dev/null +++ b/throttle_timer.go @@ -0,0 +1,57 @@ +package common + +import ( + "sync/atomic" + "time" +) + +/* +ThrottleTimer fires an event at most "dur" after each .Set() call. +If a short burst of .Set() calls happens, ThrottleTimer fires once. +If a long continuous burst of .Set() calls happens, ThrottleTimer fires +at most once every "dur". +*/ +type ThrottleTimer struct { + Name string + Ch chan struct{} + quit chan struct{} + dur time.Duration + timer *time.Timer + isSet uint32 +} + +func NewThrottleTimer(name string, dur time.Duration) *ThrottleTimer { + var ch = make(chan struct{}) + var quit = make(chan struct{}) + var t = &ThrottleTimer{Name: name, Ch: ch, dur: dur, quit: quit} + t.timer = time.AfterFunc(dur, t.fireRoutine) + t.timer.Stop() + return t +} + +func (t *ThrottleTimer) fireRoutine() { + select { + case t.Ch <- struct{}{}: + atomic.StoreUint32(&t.isSet, 0) + case <-t.quit: + // do nothing + default: + t.timer.Reset(t.dur) + } +} + +func (t *ThrottleTimer) Set() { + if atomic.CompareAndSwapUint32(&t.isSet, 0, 1) { + t.timer.Reset(t.dur) + } +} + +// For ease of .Stop()'ing services before .Start()'ing them, +// we ignore .Stop()'s on nil ThrottleTimers +func (t *ThrottleTimer) Stop() bool { + if t == nil { + return false + } + close(t.quit) + return t.timer.Stop() +} diff --git a/word.go b/word.go new file mode 100644 index 000000000..4072482b8 --- /dev/null +++ b/word.go @@ -0,0 +1,91 @@ +package common + +import ( + "bytes" + "sort" +) + +var ( + Zero256 = Word256{0} + One256 = Word256{1} +) + +type Word256 [32]byte + +func (w Word256) String() string { return string(w[:]) } +func (w Word256) TrimmedString() string { return TrimmedString(w.Bytes()) } +func (w Word256) Copy() Word256 { return w } +func (w Word256) Bytes() []byte { return w[:] } // copied. +func (w Word256) Prefix(n int) []byte { return w[:n] } +func (w Word256) Postfix(n int) []byte { return w[32-n:] } +func (w Word256) IsZero() bool { + accum := byte(0) + for _, byt := range w { + accum |= byt + } + return accum == 0 +} +func (w Word256) Compare(other Word256) int { + return bytes.Compare(w[:], other[:]) +} + +func Uint64ToWord256(i uint64) Word256 { + buf := [8]byte{} + PutUint64BE(buf[:], i) + return LeftPadWord256(buf[:]) +} + +func Int64ToWord256(i int64) Word256 { + buf := [8]byte{} + PutInt64BE(buf[:], i) + return LeftPadWord256(buf[:]) +} + +func RightPadWord256(bz []byte) (word Word256) { + copy(word[:], bz) + return +} + +func LeftPadWord256(bz []byte) (word Word256) { + copy(word[32-len(bz):], bz) + return +} + +func Uint64FromWord256(word Word256) uint64 { + buf := word.Postfix(8) + return GetUint64BE(buf) +} + +func Int64FromWord256(word Word256) int64 { + buf := word.Postfix(8) + return GetInt64BE(buf) +} + +//------------------------------------- + +type Tuple256 struct { + First Word256 + Second Word256 +} + +func (tuple Tuple256) Compare(other Tuple256) int { + firstCompare := tuple.First.Compare(other.First) + if firstCompare == 0 { + return tuple.Second.Compare(other.Second) + } else { + return firstCompare + } +} + +func Tuple256Split(t Tuple256) (Word256, Word256) { + return t.First, t.Second +} + +type Tuple256Slice []Tuple256 + +func (p Tuple256Slice) Len() int { return len(p) } +func (p Tuple256Slice) Less(i, j int) bool { + return p[i].Compare(p[j]) < 0 +} +func (p Tuple256Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Tuple256Slice) Sort() { sort.Sort(p) }