-
Notifications
You must be signed in to change notification settings - Fork 0
/
health.go
115 lines (92 loc) · 2.47 KB
/
health.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
package main
import (
"context"
"sync"
"time"
"github.com/golang/glog"
)
//reportCardGenerator, generates a report card for a healthcheck session
func reportCardGenerator(accumaltor <-chan report, th float32, pool *backendPool) {
var reportCard = make(map[string]uint8)
for x := range accumaltor {
if x.err != nil {
if _, ok := reportCard[x.backend.Name]; ok {
reportCard[x.backend.Name]++
} else {
reportCard[x.backend.Name] = 1
}
}
}
for k, v := range reportCard {
var b backend
pool.RLock()
for _, x := range pool.pool {
if x.Name == k {
b = x
break
}
}
pool.RUnlock()
if (float32(v) / float32(noOfChecks)) >= th {
//In HealthyState
pool.markHealthy(b)
glog.Info(b.Name, " found healthy in healthcheck")
} else {
pool.markUnHealthy(b)
glog.Info(b.Name, "found unhealthy in health check")
}
}
}
type report struct {
err error
backend backend
}
//healthChecker, is the health checker which runs in bakeground and checks health periodically
func healthChecker(pool *backendPool) {
pool.RLock()
//Read only part
checker := pool.healthChecker
performer := pool.healthChecker.healthCheckPerformer()
poolItems := pool.pool
pool.RUnlock()
tim, th, _ := checker.getTTI()
con := context.Background()
accumaltor := make(chan report)
go reportCardGenerator(accumaltor, th, pool)
var wg = new(sync.WaitGroup)
for _, b := range poolItems {
for i := 0; i < noOfChecks; i++ {
c, _ := context.WithTimeout(con, tim)
glog.Info("Performing health checks on backend ", b.Name)
ch := performer(c, b)
wg.Add(1)
go func(b backend) {
select {
case <-c.Done(): //It only closes when there is a Request timeout
accumaltor <- report{context.DeadlineExceeded, b}
wg.Done()
glog.Info("Health check on backend ", b.Name, " is timeout")
break
case err := <-ch: //It
accumaltor <- report{err, b}
wg.Done()
break
}
}(b)
}
}
wg.Wait() //Waiting for results to be
close(accumaltor) //As soon as the results are calculated it is the time to close the accumalator channel
}
//healthCheckService, performs health checks in background
func healthCheckService() {
pool := globalbackendPool
glog.Info("Health check service is ready")
for {
_, _, i := pool.healthChecker.getTTI()
go healthChecker(pool)
time.Sleep(i) //Wait for i duration
}
}
//globalHealthChecker, is the global health checker for instance
var globalHealthCheckerCh instanceHealthChecker