diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b1823220d..d13dfbe76 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,7 @@ * FATURE: vmagent: accept `scrape_offset` option at `scrape_config`. This option may be useful when scrapes must start at the specified offset of every scrape interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details. * FEATURE: vmauth: allow using regexp paths in `url_map`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1112) for details. +* BUGFIX: vmagent: prevent from high CPU usage bug during failing scrapes with small `scrape_timeout` (less than a few seconds). * BUGFIX: vmagent: reduce memory usage when Kubernetes service discovery is used in big number of distinct jobs by sharing the cache. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1113 * BUGFIX: prevent exponent overflow when processing extremely small values close to zero such as `2.964393875E-314`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1114 diff --git a/go.mod b/go.mod index 3809d9ecf..2d751a9e4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b - github.com/VictoriaMetrics/fasthttp v1.0.12 + github.com/VictoriaMetrics/fasthttp v1.0.13 github.com/VictoriaMetrics/metrics v1.15.2 github.com/VictoriaMetrics/metricsql v0.14.0 github.com/aws/aws-sdk-go v1.37.22 diff --git a/go.sum b/go.sum index 5af43090d..78d9a7aa1 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VictoriaMetrics/fastcache v1.5.8 h1:XW+YVx9lEXITBVv35ugK9OyotdNJVcbza69o3jmqWuI= github.com/VictoriaMetrics/fastcache v1.5.8/go.mod h1:SiMZNgwEPJ9qWLshu9tyuE6bKc9ZWYhcNV/L7jurprQ= -github.com/VictoriaMetrics/fasthttp v1.0.12 h1:Ag0E119yrH4BTxVyjKD9TeiSImtG9bUcg/stItLJhSE= -github.com/VictoriaMetrics/fasthttp v1.0.12/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU= +github.com/VictoriaMetrics/fasthttp v1.0.13 h1:5JNS4vSPdN4QyfcpAg3Y1Wznf0uXEuSOFpeIlFw3MgM= +github.com/VictoriaMetrics/fasthttp v1.0.13/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrtNHXBJR6N376EgiSHU= github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.15.2 h1:w/GD8L9tm+gvx1oZvAofRRXwammiicdI0jgLghA2Gdo= github.com/VictoriaMetrics/metrics v1.15.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= diff --git a/vendor/github.com/VictoriaMetrics/fasthttp/client.go b/vendor/github.com/VictoriaMetrics/fasthttp/client.go index 5f1f433d6..60d1f19b7 100644 --- a/vendor/github.com/VictoriaMetrics/fasthttp/client.go +++ b/vendor/github.com/VictoriaMetrics/fasthttp/client.go @@ -1023,7 +1023,7 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) panic("BUG: resp cannot be nil") } - atomic.StoreUint32(&c.lastUseTime, uint32(CoarseTimeNow().Unix()-startTimeUnix)) + atomic.StoreUint32(&c.lastUseTime, uint32(time.Now().Unix()-startTimeUnix)) // Free up resources occupied by response before sending the request, // so the GC may reclaim these resources (e.g. response body). @@ -1039,7 +1039,7 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) // Optimization: update write deadline only if more than 25% // of the last write deadline exceeded. // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() + currentTime := time.Now() if currentTime.Sub(cc.lastWriteDeadlineTime) > (c.WriteTimeout >> 2) { if err = conn.SetWriteDeadline(currentTime.Add(c.WriteTimeout)); err != nil { c.closeConn(cc) @@ -1083,7 +1083,7 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) // Optimization: update read deadline only if more than 25% // of the last read deadline exceeded. // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() + currentTime := time.Now() if currentTime.Sub(cc.lastReadDeadlineTime) > (c.ReadTimeout >> 2) { if err = conn.SetReadDeadline(currentTime.Add(c.ReadTimeout)); err != nil { c.closeConn(cc) @@ -1255,7 +1255,7 @@ func acquireClientConn(conn net.Conn) *clientConn { } cc := v.(*clientConn) cc.c = conn - cc.createdTime = CoarseTimeNow() + cc.createdTime = time.Now() return cc } @@ -1267,7 +1267,7 @@ func releaseClientConn(cc *clientConn) { var clientConnPool sync.Pool func (c *HostClient) releaseConn(cc *clientConn) { - cc.lastUseTime = CoarseTimeNow() + cc.lastUseTime = time.Now() c.connsLock.Lock() c.conns = append(c.conns, cc) c.connsLock.Unlock() @@ -1970,7 +1970,7 @@ func (c *pipelineConnClient) writer(conn net.Conn, stopCh <-chan struct{}) error // Optimization: update write deadline only if more than 25% // of the last write deadline exceeded. // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() + currentTime := time.Now() if currentTime.Sub(lastWriteDeadlineTime) > (writeTimeout >> 2) { if err = conn.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { w.err = err @@ -2051,7 +2051,7 @@ func (c *pipelineConnClient) reader(conn net.Conn, stopCh <-chan struct{}) error // Optimization: update read deadline only if more than 25% // of the last read deadline exceeded. // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() + currentTime := time.Now() if currentTime.Sub(lastReadDeadlineTime) > (readTimeout >> 2) { if err = conn.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { w.err = err diff --git a/vendor/github.com/VictoriaMetrics/fasthttp/coarseTime.go b/vendor/github.com/VictoriaMetrics/fasthttp/coarseTime.go deleted file mode 100644 index 03c14f39a..000000000 --- a/vendor/github.com/VictoriaMetrics/fasthttp/coarseTime.go +++ /dev/null @@ -1,28 +0,0 @@ -package fasthttp - -import ( - "sync/atomic" - "time" -) - -// CoarseTimeNow returns the current time truncated to the nearest second. -// -// This is a faster alternative to time.Now(). -func CoarseTimeNow() time.Time { - tp := coarseTime.Load().(*time.Time) - return *tp -} - -func init() { - t := time.Now().Truncate(time.Second) - coarseTime.Store(&t) - go func() { - for { - time.Sleep(time.Second) - t := time.Now().Truncate(time.Second) - coarseTime.Store(&t) - } - }() -} - -var coarseTime atomic.Value diff --git a/vendor/github.com/VictoriaMetrics/fasthttp/server.go b/vendor/github.com/VictoriaMetrics/fasthttp/server.go index 3e74369e7..c2432c840 100644 --- a/vendor/github.com/VictoriaMetrics/fasthttp/server.go +++ b/vendor/github.com/VictoriaMetrics/fasthttp/server.go @@ -1281,7 +1281,7 @@ func (s *Server) Serve(ln net.Listener) error { if time.Since(lastOverflowErrorTime) > time.Minute { s.logger().Printf("The incoming connection cannot be served, because %d concurrent connections are served. "+ "Try increasing Server.Concurrency", maxWorkersCount) - lastOverflowErrorTime = CoarseTimeNow() + lastOverflowErrorTime = time.Now() } // The current server reached concurrency limit, @@ -1323,7 +1323,7 @@ func acceptConn(s *Server, ln net.Listener, lastPerIPErrorTime *time.Time) (net. if time.Since(*lastPerIPErrorTime) > time.Minute { s.logger().Printf("The number of connections from %s exceeds MaxConnsPerIP=%d", getConnIP4(c), s.MaxConnsPerIP) - *lastPerIPErrorTime = CoarseTimeNow() + *lastPerIPErrorTime = time.Now() } continue } @@ -1438,7 +1438,7 @@ func (s *Server) serveConn(c net.Conn) error { serverName := s.getServerName() connRequestNum := uint64(0) connID := nextConnID() - currentTime := CoarseTimeNow() + currentTime := time.Now() connTime := currentTime maxRequestBodySize := s.MaxRequestBodySize if maxRequestBodySize <= 0 { @@ -1491,7 +1491,7 @@ func (s *Server) serveConn(c net.Conn) error { } } - currentTime = CoarseTimeNow() + currentTime = time.Now() ctx.lastReadDuration = currentTime.Sub(ctx.time) if err != nil { @@ -1632,7 +1632,7 @@ func (s *Server) serveConn(c net.Conn) error { break } - currentTime = CoarseTimeNow() + currentTime = time.Now() } if br != nil { @@ -1688,7 +1688,7 @@ func (s *Server) updateWriteDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTi // Optimization: update write deadline only if more than 25% // of the last write deadline exceeded. // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() + currentTime := time.Now() if currentTime.Sub(lastDeadlineTime) > (writeTimeout >> 2) { if err := c.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { panic(fmt.Sprintf("BUG: error in SetWriteDeadline(%s): %s", writeTimeout, err)) @@ -1871,7 +1871,7 @@ func (ctx *RequestCtx) Init2(conn net.Conn, logger Logger, reduceMemoryUsage boo ctx.connID = nextConnID() ctx.s = fakeServer ctx.connRequestNum = 0 - ctx.connTime = CoarseTimeNow() + ctx.connTime = time.Now() ctx.time = ctx.connTime keepBodyBuffer := !reduceMemoryUsage diff --git a/vendor/github.com/VictoriaMetrics/fasthttp/workerpool.go b/vendor/github.com/VictoriaMetrics/fasthttp/workerpool.go index cf602e050..081ac65c3 100644 --- a/vendor/github.com/VictoriaMetrics/fasthttp/workerpool.go +++ b/vendor/github.com/VictoriaMetrics/fasthttp/workerpool.go @@ -187,7 +187,7 @@ func (wp *workerPool) getCh() *workerChan { } func (wp *workerPool) release(ch *workerChan) bool { - ch.lastUseTime = CoarseTimeNow() + ch.lastUseTime = time.Now() wp.lock.Lock() if wp.mustStop { wp.lock.Unlock() diff --git a/vendor/modules.txt b/vendor/modules.txt index 5c3949879..f1e50bf9d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -13,7 +13,7 @@ cloud.google.com/go/storage # github.com/VictoriaMetrics/fastcache v1.5.8 ## explicit github.com/VictoriaMetrics/fastcache -# github.com/VictoriaMetrics/fasthttp v1.0.12 +# github.com/VictoriaMetrics/fasthttp v1.0.13 ## explicit github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil