better proftrigger situation

Dieterbe · Dieterbe · commit febea9dfcddf · 2016-10-07T18:40:39.000+03:00
* use new version which uses bytes consumed by process see Dieterbe/profiletrigger@e8a1450 * set default threshold of 25GB which should work great for 32GB systems
diff --git a/docs/config.md b/docs/config.md
@@ -109,7 +109,7 @@ proftrigger-path = /tmp
 # minimum time between triggered profiles
 proftrigger-min-diff = 1h
 # if this many bytes allocated, trigger a heap profile
-proftrigger-heap-thresh = 10000000
+proftrigger-heap-thresh = 25000000000
 # only log incoming requests if their timerange is at least this duration. Use 0 to disable
 log-min-dur = 5min
 # only log log-level and higher. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -14,11 +14,9 @@ your (infrequent) updates.  The primary won't add them to its in-memory chunks,
 a counter of total amount of bytes allocated during process lifetime. (incl freed data)
 * `bytes_alloc.not_freed`:  
 a gauge of currently allocated (within the runtime) memory.
-it does not include freed data and drops at every GC run.
-this is what is inspected by the profiletrigger
-note that total memory used by the process can be about 2x this.
+it does not include freed data so it drops at every GC run.
 * `bytes_sys`:  
-the amount of bytes currently obtained from the system
+the amount of bytes currently obtained from the system by the process.  This is what the profiletrigger looks at.
 * `cluster.promotion_wait`:  
 how long a candidate (secondary node) has to wait until it can become a primary
 When the timer becomes 0 it means the in-memory buffer has been able to fully populate so that if you stop a primary
diff --git a/docs/operations.md b/docs/operations.md
@@ -41,7 +41,10 @@ Metrictank crashed. What to do?
 1) Check `dmesg` to see if it was killed by the kernel, maybe it was consuming too much RAM
    If it was, check the grafana dashboard which may explain why. (sudden increase in ingested data? increase in requests or the amount of data requested? slow requests?)
    Tips:
-   * The [profiletrigger](https://github.com/raintank/metrictank/blob/master/docs/config.md#profiling-instrumentation-and-logging) functionality can automatically trigger a memory profile and save it to disk.  This can be very helpful if suddently memory usage spikes up and then metrictank gets killed in seconds or minutes.  It helps diagnose problems in the code base that may lead to memory savings.  The profiletrigger looks at the `bytes_alloc.not_freed` metric which is just memory allocated within the runtime.  Amount of memory consumed by the process may be twice this.  So as a rule of thumb, if you have a system running just metrictank, set it to about 40% of available memory.
+   * The [profiletrigger](https://github.com/raintank/metrictank/blob/master/docs/config.md#profiling-instrumentation-and-logging) functionality can automatically trigger
+   a memory profile and save it to disk.  This can be very helpful if suddently memory usage spikes up and then metrictank gets killed in seconds or minutes.  
+   It helps diagnose problems in the codebase that may lead to memory savings.  The profiletrigger looks at the `bytes_sys` metric which is
+   the amount of memory consumed by the process.
    * Use [rollups](https://github.com/raintank/metrictank/blob/master/docs/consolidation.md#rollups) to be able to answer queries for long timeframes with less data
 2) Check the metrictank log.
    If it exited due to a panic, you should probably open a [ticket](https://github.com/raintank/metrictank/issues) with the output of `metrictank --version`, the panic, and perhaps preceeding log data.
diff --git a/metrictank-sample.ini b/metrictank-sample.ini
@@ -94,8 +94,9 @@ proftrigger-freq = 60s
 proftrigger-path = /tmp
 # minimum time between triggered profiles
 proftrigger-min-diff = 1h
-# if this many bytes allocated, trigger a heap profile
-proftrigger-heap-thresh = 10000000
+# if process consumes this many bytes (see bytes_sys in dashboard), trigger a heap profile for developer diagnosis
+# set it higher than your typical memory usage, but lower than how much RAM the process can take before its get killed
+proftrigger-heap-thresh = 25000000000
 
 # only log incoming requests if their timerange is at least this duration. Use 0 to disable
 log-min-dur = 5min
diff --git a/metrictank.go b/metrictank.go
@@ -105,7 +105,7 @@ var (
 	proftrigPath       = flag.String("proftrigger-path", "/tmp", "path to store triggered profiles")
 	proftrigFreqStr    = flag.String("proftrigger-freq", "60s", "inspect status frequency. set to 0 to disable")
 	proftrigMinDiffStr = flag.String("proftrigger-min-diff", "1h", "minimum time between triggered profiles")
-	proftrigHeapThresh = flag.Int("proftrigger-heap-thresh", 10000000, "if this many bytes allocated, trigger a profile")
+	proftrigHeapThresh = flag.Int("proftrigger-heap-thresh", 25000000000, "if this many bytes allocated, trigger a profile")
 
 	logMinDurStr = flag.String("log-min-dur", "5min", "only log incoming requests if their timerange is at least this duration. Use 0 to disable")
 
@@ -124,13 +124,11 @@ var (
 	points            met.Gauge
 
 	// metric bytes_alloc.not_freed is a gauge of currently allocated (within the runtime) memory.
-	// it does not include freed data and drops at every GC run.
-	// this is what is inspected by the profiletrigger
-	// note that total memory used by the process can be about 2x this.
+	// it does not include freed data so it drops at every GC run.
 	alloc met.Gauge
 	// metric bytes_alloc.incl_freed is a counter of total amount of bytes allocated during process lifetime. (incl freed data)
 	totalAlloc met.Gauge
-	// metric bytes_sys is the amount of bytes currently obtained from the system
+	// metric bytes_sys is the amount of bytes currently obtained from the system by the process.  This is what the profiletrigger looks at.
 	sysBytes       met.Gauge
 	clusterPrimary met.Gauge
 
diff --git a/scripts/config/metrictank-docker.ini b/scripts/config/metrictank-docker.ini
@@ -92,7 +92,7 @@ proftrigger-path = /tmp
 # minimum time between triggered profiles
 proftrigger-min-diff = 1h
 # if this many bytes allocated, trigger a heap profile
-proftrigger-heap-thresh = 10000000
+proftrigger-heap-thresh = 25000000000
 
 # only log incoming requests if their timerange is at least this duration. Use 0 to disable
 log-min-dur = 5min
diff --git a/scripts/config/metrictank-package.ini b/scripts/config/metrictank-package.ini
@@ -92,7 +92,7 @@ proftrigger-path = /tmp
 # minimum time between triggered profiles
 proftrigger-min-diff = 1h
 # if this many bytes allocated, trigger a heap profile
-proftrigger-heap-thresh = 10000000
+proftrigger-heap-thresh = 25000000000
 
 # only log incoming requests if their timerange is at least this duration. Use 0 to disable
 log-min-dur = 5min
diff --git a/vendor/github.com/Dieterbe/profiletrigger/heap/heap.go b/vendor/github.com/Dieterbe/profiletrigger/heap/heap.go
diff --git a/vendor/vendor.json b/vendor/vendor.json
@@ -9,10 +9,11 @@
 			"revisionTime": "2015-09-30T14:07:41Z"
 		},
 		{
-			"checksumSHA1": "GbgPwWPwxTliU7wwIwp93GcU+E4=",
+			"checksumSHA1": "P8h2SWEK3NsJleSPe+mVNNpLG6Y=",
+			"origin": "github.com/raintank/metrictank/vendor/github.com/Dieterbe/profiletrigger/heap",
 			"path": "github.com/Dieterbe/profiletrigger/heap",
-			"revision": "49951b329b2f0508075c2e8505599470bf7b20e3",
-			"revisionTime": "2016-05-24T13:14:35Z"
+			"revision": "d90c4b0cfeed756381675e85cc6e6b8a02cb01a6",
+			"revisionTime": "2016-10-07T15:24:48Z"
 		},
 		{
 			"checksumSHA1": "AAXMx9vb6vmVZF2ieqNepAfeJFM=",
