From b08f51f5d306773a3fb2274a6213933b08ae8943 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@victoriametrics.com>
Date: Fri, 22 Oct 2021 16:44:41 +0300
Subject: [PATCH] lib/promscrape: do not populate response body to memory in
 stream parsing mode if -promscrape.noStaleMarkers is set

The response body isn't used if -promscrape.noStaleMarkers is set after the commit 2876137c928413fa8f3641a86a09d05d062a60aa ,
so there is no sense in pupulating it in memory. This should reduce memory usage when scraping big responses.

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1728#issuecomment-949630694
---
 lib/promscrape/scrapework.go | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go
index 3d88a18a8..0563cc228 100644
--- a/lib/promscrape/scrapework.go
+++ b/lib/promscrape/scrapework.go
@@ -453,13 +453,18 @@ func (sw *scrapeWork) pushData(wr *prompbmarshal.WriteRequest) {
 }
 
 type streamBodyReader struct {
-	sr   *streamReader
-	body []byte
+	sr          *streamReader
+	body        []byte
+	bodyLen     int
+	captureBody bool
 }
 
 func (sbr *streamBodyReader) Read(b []byte) (int, error) {
 	n, err := sbr.sr.Read(b)
-	sbr.body = append(sbr.body, b[:n]...)
+	sbr.bodyLen += n
+	if sbr.captureBody {
+		sbr.body = append(sbr.body, b[:n]...)
+	}
 	return n, err
 }
 
@@ -468,7 +473,9 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	samplesPostRelabeling := 0
 	wc := writeRequestCtxPool.Get(sw.prevLabelsLen)
 	// Do not pool sbr and do not pre-allocate sbr.body in order to reduce memory usage when scraping big responses.
-	sbr := &streamBodyReader{}
+	sbr := &streamBodyReader{
+		captureBody: !*noStaleMarkers,
+	}
 
 	sr, err := sw.GetStreamReader()
 	if err != nil {
@@ -507,7 +514,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	endTimestamp := time.Now().UnixNano() / 1e6
 	duration := float64(endTimestamp-realTimestamp) / 1e3
 	scrapeDuration.Update(duration)
-	scrapeResponseSize.Update(float64(len(bodyString)))
+	scrapeResponseSize.Update(float64(sbr.bodyLen))
 	up := 1
 	if err != nil {
 		if samplesScraped == 0 {
@@ -530,7 +537,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
 	sw.addAutoTimeseries(wc, "scrape_timeout_seconds", sw.Config.ScrapeTimeout.Seconds(), scrapeTimestamp)
 	sw.pushData(&wc.writeRequest)
 	sw.prevLabelsLen = len(wc.labels)
-	sw.prevBodyLen = len(bodyString)
+	sw.prevBodyLen = sbr.bodyLen
 	wc.reset()
 	writeRequestCtxPool.Put(wc)
 	if !areIdenticalSeries {