diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index fbd4ffe23..2cd7d371f 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -506,6 +506,10 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) { metrics.WriteGaugeUint64(w, `vm_parts{type="indexdb/inmemory"}`, idbm.InmemoryPartsCount) metrics.WriteGaugeUint64(w, `vm_parts{type="indexdb/file"}`, idbm.FilePartsCount) + metrics.WriteGaugeUint64(w, `vm_last_partition_parts{type="storage/inmemory"}`, tm.LastPartition.InmemoryPartsCount) + metrics.WriteGaugeUint64(w, `vm_last_partition_parts{type="storage/small"}`, tm.LastPartition.SmallPartsCount) + metrics.WriteGaugeUint64(w, `vm_last_partition_parts{type="storage/big"}`, tm.LastPartition.BigPartsCount) + metrics.WriteGaugeUint64(w, `vm_blocks{type="storage/inmemory"}`, tm.InmemoryBlocksCount) metrics.WriteGaugeUint64(w, `vm_blocks{type="storage/small"}`, tm.SmallBlocksCount) metrics.WriteGaugeUint64(w, `vm_blocks{type="storage/big"}`, tm.BigBlocksCount) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7b8c8c0c7..3c46f4d44 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,8 @@ The sandbox cluster installation is running under the constant load generated by ## tip +* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): expose `vm_last_partition_parts` [metrics](https://docs.victoriametrics.com/#monitoring), which show the number of [parts in the latest partition](https://docs.victoriametrics.com/#storage). These metrics may help debugging query performance slowdown related to the increased number of parts in the last partition, since usually all the ingested data is written to the last partition and all the queries are performed over the recently ingested data, e.g. the last partition. + ## [v1.98.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.98.0) Released at 2024-02-14 diff --git a/lib/storage/table.go b/lib/storage/table.go index 1dafbbc87..9a780d2ea 100644 --- a/lib/storage/table.go +++ b/lib/storage/table.go @@ -217,17 +217,34 @@ func (tb *table) NotifyReadWriteMode() { type TableMetrics struct { partitionMetrics + // LastPartition contains metrics for the last partition. + // These metrics are important, since the majority of data ingestion + // and querying goes to the last partition. + LastPartition partitionMetrics + PartitionsRefCount uint64 } // UpdateMetrics updates m with metrics from tb. func (tb *table) UpdateMetrics(m *TableMetrics) { - tb.ptwsLock.Lock() - for _, ptw := range tb.ptws { + ptws := tb.GetPartitions(nil) + defer tb.PutPartitions(ptws) + + for _, ptw := range ptws { ptw.pt.UpdateMetrics(&m.partitionMetrics) m.PartitionsRefCount += atomic.LoadUint64(&ptw.refCount) } - tb.ptwsLock.Unlock() + + // Collect separate metrics for the last partition. + if len(ptws) > 0 { + ptwLast := ptws[0] + for _, ptw := range ptws[1:] { + if ptw.pt.tr.MinTimestamp > ptwLast.pt.tr.MinTimestamp { + ptwLast = ptw + } + } + ptwLast.pt.UpdateMetrics(&m.LastPartition) + } } // ForceMergePartitions force-merges partitions in tb with names starting from the given partitionNamePrefix.