Commit a228ea48 authored by Josh Roppo's avatar Josh Roppo

Loges now counts the number of throttled log messages for a log key

The throttled count is stored on the Throttle struct which is associated
via the global log throttle map. When the throttled messages are allowed
to log the count of messages which were hidden is prefixed to the
message.
parent 5e911090
......@@ -275,11 +275,16 @@ func LogThrottleKey(logLvl, limit int, key, format string, v ...interface{}) {
th = NewThrottler(limit, 3600*time.Second)
logThrottles[key] = th
}
if th.Throttle() {
skip, throttleCount := th.Throttle()
if skip {
throttleMu.Unlock()
return
}
throttleMu.Unlock()
if throttleCount > 0 {
format = fmt.Sprintf("LogsThrottled[%d] %s", throttleCount, format)
}
DoLog(3, logLvl, fmt.Sprintf(format, v...))
}
}
......@@ -297,11 +302,17 @@ func LogThrottle(logLvl, limit int, format string, v ...interface{}) {
th = NewThrottler(limit, 3600*time.Second)
logThrottles[format] = th
}
if th.Throttle() {
var throttleCount int32
skip, throttleCount := th.Throttle()
if skip {
throttleMu.Unlock()
return
}
throttleMu.Unlock()
if throttleCount > 0 {
format = fmt.Sprintf("LogsThrottled[%d] %s", throttleCount, format)
}
DoLog(3, logLvl, fmt.Sprintf(format, v...))
}
}
......@@ -319,11 +330,14 @@ func LogThrottleD(depth, logLvl, limit int, format string, v ...interface{}) {
th = NewThrottler(limit, 3600*time.Second)
logThrottles[format] = th
}
if th.Throttle() {
skip, throttleCount := th.Throttle()
if skip {
throttleMu.Unlock()
return
}
throttleMu.Unlock()
format = fmt.Sprintf("Log Throttled[%d] %s", throttleCount, format)
DoLog(depth, logLvl, fmt.Sprintf(format, v...))
}
}
......
......@@ -9,6 +9,7 @@ type Throttler struct {
// Limit to this events/per
maxPer float64
per float64
count int32
// Last Event
last time.Time
......@@ -24,16 +25,19 @@ func NewThrottler(max int, per time.Duration) *Throttler {
return &Throttler{
maxPer: float64(max),
allowance: float64(max),
count: int32(0),
last: time.Now(),
per: per.Seconds(),
}
}
// Should we limit this because we are above rate?
func (r *Throttler) Throttle() bool {
// Returns a bool of whether to throttle the message, and a count
// of previous log messages throttled since last log message.
func (r *Throttler) Throttle() (bool, int32) {
if r.maxPer == 0 {
return false
return false, 0
}
// http://stackoverflow.com/questions/667508/whats-a-good-rate-limiting-algorithm
......@@ -48,9 +52,17 @@ func (r *Throttler) Throttle() bool {
}
if r.allowance <= 1.0 {
return true // do throttle/limit
r.count++ // increment throttled log count
return true, r.count // do throttle/limit
}
tmpCount := r.count
r.count = 0 // reset count
r.allowance -= 1.0
return false // dont throttle
return false, tmpCount // dont throttle, return previous throttle count
}
func (r *Throttler) ThrottleCount() int32 {
return r.count
}
......@@ -10,21 +10,63 @@ import (
func TestThrottleer(t *testing.T) {
th := NewThrottler(10, 10*time.Second)
for i := 0; i < 10; i++ {
assert.Tf(t, th.Throttle() == false, "Should not throttle %v", i)
thb, tc := th.Throttle()
assert.Tf(t, thb == false, "Should not throttle %v", i)
assert.Tf(t, tc < 10, "Throttle count should remain below 10 %v", tc)
time.Sleep(time.Millisecond * 10)
}
throttled := 0
th = NewThrottler(10, 1*time.Second)
// We are going to loop 20 times, first 10 should make it, next 10 throttled
for i := 0; i < 20; i++ {
LogThrottleKey(WARN, 10, "throttle", "hello %v", i)
if th.Throttle() {
thb, tc := th.Throttle()
if thb {
throttled += 1
assert.Tf(t, int(tc) == i-9, "Throttle count should rise %v, i: %d", tc, i)
}
}
assert.Tf(t, throttled == 10, "Should throttle 10 of 20 requests: %v", throttled)
// Now sleep for 1 second so that we should
// no longer be throttled
time.Sleep(time.Second * 2)
thb, _ := th.Throttle()
assert.Tf(t, thb == false, "We should not have been throttled")
}
func TestThrottler2(t *testing.T) {
th := NewThrottler(10, 1*time.Second)
tkey := "throttle2"
throttleMu.Lock()
logThrottles[tkey] = th
throttleMu.Unlock()
th, ok := logThrottles[tkey]
if !ok {
t.Errorf("Throttle key %s not created!", tkey)
}
// We are going to loop 20 times, first 10 should make it, next 10 throttled
for i := 0; i < 20; i++ {
LogThrottleKey(WARN, 10, tkey, "hello %v", i)
}
throttleMu.Lock()
th = logThrottles[tkey]
tcount := th.ThrottleCount()
assert.Tf(t, tcount == 10, "Should throttle 10 of 20 requests: %v", tcount)
throttleMu.Unlock()
// Now sleep for 1 second so that we should
// no longer be throttled
time.Sleep(time.Second * 1)
assert.Tf(t, th.Throttle() == false, "We should not have been throttled")
LogThrottleKey(WARN, 10, tkey, "hello again %v", 20)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment