Receive: high in flight requests and high context deadline exceeded and ingestion latency in main branch about thanos HOT 9 CLOSED

[
    {
        "endpoints": [
            {
                "address": "thanos-receive-rep0-0.thanos-receive-svc:10901",
                "az": "zone-0"
            },
            {
                "address": "thanos-receive-rep0-1.thanos-receive-svc:10901",
                "az": "zone-0"
            },
            {
                "address": "thanos-receive-rep1-0.thanos-receive-svc:10901",
                "az": "zone-1"
            },
            {
                "address": "thanos-receive-rep1-1.thanos-receive-svc:10901",
                "az": "zone-1"
            },
            {
                "address": "thanos-receive-rep2-0.thanos-receive-svc:10901",
                "az": "zone-2"
            },
            {
                "address": "thanos-receive-rep2-1.thanos-receive-svc:10901",
                "az": "zone-2"
            }
        ],
        "hashring": "thanos-receive"
    }
]

	// Do the writes to remote nodes. Run them all in parallel.
	for writeDestination := range remoteWrites {
		h.sendRemoteWrite(ctx, params.tenant, writeDestination, remoteWrites[writeDestination], params.alreadyReplicated, responses, wg)
	}

func (p *peerWorker) RemoteWriteAsync(ctx context.Context, req *storepb.WriteRequest, er endpointReplica, seriesIDs []int, responseWriter chan writeResponse, cb func(error)) {
	p.initWorkers()

	w := peerWorkItem{
		cc:          p.cc,
		req:         req,
		workResult:  make(chan peerWorkResponse, 1),
		workItemCtx: ctx,
		er:          er,

		sendTime: time.Now(),
	}

	p.work <- w
	res := <-w.workResult

	responseWriter <- newWriteResponse(seriesIDs, res.err, er)
	cb(res.err)
}