Commit 7d3d6746 authored by Aaron Raddon's avatar Aaron Raddon

new log throttler

parent 10141057
......@@ -6,6 +6,7 @@ import (
"os"
"runtime"
"strings"
"sync"
)
const (
......@@ -57,6 +58,8 @@ var (
}
postFix = "" //\033[0m
LogLevelWords map[string]int = map[string]int{"fatal": 0, "error": 1, "warn": 2, "info": 3, "debug": 4, "none": -1}
logThrottles = make(map[string]*Throttler)
throttleMu sync.Mutex
)
// Setup default logging to Stderr, equivalent to:
......@@ -224,6 +227,28 @@ func LogTracef(logLvl int, format string, v ...interface{}) {
}
}
// Throttle logging based on key, such that key would never occur more than
// @limit times per minute
//
// LogThrottleKey(u.ERROR, 1,"error_that_happens_a_lot" "message %s", varx)
//
func LogThrottleKey(logLvl, limit int, key, format string, v ...interface{}) {
if LogLevel >= logLvl {
throttleMu.Lock()
th, ok := logThrottles[key]
if !ok {
th = NewThrottler(limit, 60)
logThrottles[key] = th
}
if th.Throttle() {
throttleMu.Unlock()
return
}
throttleMu.Unlock()
DoLog(4, logLvl, fmt.Sprintf(format, v...))
}
}
// Log to logger if setup
// Logf(ERROR, "message %d", 20)
func Logf(logLvl int, format string, v ...interface{}) {
......
package gou
import (
"time"
)
type Throttler struct {
// Limit to this events/second
maxPerSec int
// Last Event
last time.Time
// How many events are allowed left to happen?
// Starts at limit, decrements down
allowance float64
per float64
}
// new Throttler that will tell you to limit or not based
// on given max events per second input @limit
func NewThrottler(maxPerSecond, per int) *Throttler {
return &Throttler{
maxPerSec: maxPerSecond,
allowance: float64(maxPerSecond),
last: time.Now(),
per: float64(per),
}
}
// Should we limit this because we are above rate?
func (r *Throttler) Throttle() bool {
if r.maxPerSec == 0 {
return false
}
// http://stackoverflow.com/questions/667508/whats-a-good-rate-limiting-algorithm
rate := float64(r.maxPerSec)
now := time.Now()
elapsed := float64(now.Sub(r.last).Nanoseconds()) / 1e9 // nano Seconds
r.last = now
r.allowance += elapsed * (rate / r.per)
//Infof("maxRate: %v cur: %v elapsed:%-6.6f incr: %v", r.maxPerSec, int(r.allowance), elapsed, elapsed*float64(r.maxPerSec))
if r.allowance > rate {
r.allowance = float64(r.maxPerSec)
}
if r.allowance <= 1.0 {
return true // do throttle/limit
}
r.allowance -= 1.0
return false // dont throttle
}
/*
type Ratelimiter struct {
rate int // conn/sec
last time.Time // last time we were polled/asked
allowance float64
}
// Create new rate limiter that limits at rate/sec
func NewRateLimiter(rate int) (*Ratelimiter, error) {
r := Ratelimiter{rate:rate, last:time.Now()}
r.allowance = float64(r.rate)
return &r, nil
}
// Return true if the current call exceeds the set rate, false
// otherwise
func (r* Ratelimiter) Limit() bool {
// handle cases where rate in config file is unset - defaulting
// to "0" (unlimited)
if r.rate == 0 {
return false
}
rate := float64(r.rate)
now := time.Now()
elapsed := now.Sub(r.last)
r.last = now
r.allowance += float64(elapsed) * rate
// Clamp number of tokens in the bucket. Don't let it get
// unboundedly large
if r.allowance > rate {
r.allowance = rate
}
var ret bool
if r.allowance < 1.0 {
ret = true
} else {
r.allowance -= 1.0
ret = false
}
return ret
}
*/
package gou
import (
"testing"
"time"
"github.com/bmizerany/assert"
)
func TestThrottleer(t *testing.T) {
th := NewThrottler(10, 10)
for i := 0; i < 10; i++ {
assert.Tf(t, th.Throttle() == false, "Should not throttle %v", i)
time.Sleep(time.Millisecond * 10)
}
throttled := 0
th = NewThrottler(10, 1)
// We are going to loop 20 times, first 10 should make it, next 10 throttled
for i := 0; i < 20; i++ {
if th.Throttle() {
throttled += 1
}
}
assert.Tf(t, throttled == 10, "Should throttle 10 of 20 requests: %v", throttled)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment