Repository navigation
Description
Bug summary
Memory leak from repeated live plotting: seems to be caused by transform._parents growing continuously over time, as in #11956 and #11972.
Code for reproduction
import datetime
#import tkinter for GUI
import tkinter as tk
from tkinter import W, LEFT
#font types
TITLE_FONT = ("Verdana", 14, 'bold')
LARGE_FONT = ("Verdana", 12)
MEDIUM_FONT = ("Verdana", 10)
SMALL_FONT = ("Verdana", 8)
#import stuff for graph
import matplotlib
from matplotlib import ticker as mticker
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk
matplotlib.use('TkAgg')
from matplotlib import figure
from matplotlib import dates as mdates
#import animation to make graph live
import matplotlib.animation as animation
from matplotlib import style
style.use("seaborn-darkgrid")
import random as r
num_contacts = 5
import cProfile
import pstats
import io
#create figure for plots and set figure size/layout
f = figure.Figure(figsize=(16.6,15), dpi=100, facecolor='white')
f.subplots_adjust(top=0.993, bottom=0.015, left=0.04, right = 0.96, hspace=0.65)
param_dict = {}
param_list = ['pH', 'TDS (ppm)', 'Rela. Humidity (%)', 'Air Temp (\N{DEGREE SIGN}C)', 'Water Temp (\N{DEGREE SIGN}C)', 'Water Level (cm)']
param_ylim = [(0, 15), (0, 15), (0, 15), (0, 15), (0, 15), (0, 15)]
live_dict = {}
class Live_Text:
def __init__(self, label):
self.label = label
class Sensor_Plot:
def __init__(self, plot, tList, x_ax, ylim, param, incoming_data, plot_color):
self.plot = plot
self.tList = tList
self.x_ax = x_ax
self.ylim = ylim
self.param = param
self.incoming_data = incoming_data #<- graph is bound by incoming data and Data Summary Table displays most recent value 20 of them
self.plot_color = plot_color #initially 'b' for all
def make_plot(self):
self.plot.clear()
self.plot.set_xlabel('Time')
self.plot.set_ylabel(self.param)
self.plot.set_ylim(self.ylim)
self.x_ax.xaxis_date()
self.x_ax.xaxis.set_major_formatter(mdates.DateFormatter('%a %I:%M:%S %p'))
[tk.set_visible(True) for tk in self.x_ax.get_xticklabels()]
[label.set_rotation(10) for label in self.x_ax.xaxis.get_ticklabels()] #slant the x axis tick labels for extra coolness
if len(self.tList) > 4:
self.x_ax.set_xlim(self.tList[-2], self.tList[0])
self.x_ax.xaxis.set_major_locator(mticker.MaxNLocator(nbins = 4))
self.plot.fill_between(self.tList, self.incoming_data, #where=(self.incoming_data > [0]*len(self.incoming_data))
facecolor=self.plot_color, edgecolor=self.plot_color, alpha=0.5) #blue @initilization
def initialize_plots(): #intiailizes plots...
global initialize_plots
try:
most_recent = [(int(round(datetime.datetime.now().timestamp())),) + tuple([r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10)])]
for i, param in enumerate(param_list, 1):
tList = []
most_recent_any_size = []
for j in range(len(most_recent)):
#time_f = datetime.strptime(most_recent[j][0], "%m/%d/%Y %H:%M:%S")
time_f = datetime.datetime.fromtimestamp(most_recent[j][0])
tList.append(time_f)
most_recent_any_size.append(most_recent[j][i])
subplot = f.add_subplot(6, 2, i) # sharex?
x_ax = f.get_axes()
current_plot = Sensor_Plot(subplot, tList, x_ax[i-1], param_ylim[i-1], param, most_recent_any_size, 'b')
param_dict[param] = current_plot
current_plot.make_plot()
except: #if there is no data points available to plot, initialize the subplots
for i, param in enumerate(param_list, 1):
subplot = f.add_subplot(6, 2, i)
x_ax = f.get_axes()
current_plot = Sensor_Plot(subplot, [], x_ax[i-1], param_ylim[i-1], param, [], 'b')
param_dict[param] = current_plot
#current_plot.make_plot()
#reader.commit()
initialize_plots = _plots_initialized
def _plots_initialized(): #ensures plots only intialized once though!
pass
initialize_plots()
###ANIMATE FUNCTION, REMOVE LAST ITEM FROM MOST_RECENT_ANY LIST AND INSERT FRESHLY CALLED VALUE TO BE FIRST IN LIST
def animate(ii):
profile = cProfile.Profile()
profile.enable()
while True:
most_recent_time_graphed = param_dict[param_list[0]] #first, pulls up first plot
#most_recent = reader.query_by_num(table="SensorData", num=1)
#generate fake sensor data here
most_recent = [(int(round(datetime.datetime.now().timestamp())),) + tuple([r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10), r.randint(1,10)])] #log time in unix as int
print(most_recent)
#reader.commit() #if identical, do not animate
#then checks that plot's time list
if (len(most_recent) == 0):
break
#time_reader = datetime.strptime(most_recent[0][0], "%m/%d/%Y %H:%M:%S")
time_reader = datetime.datetime.fromtimestamp(most_recent[0][0])
if (len(most_recent_time_graphed.tList) != 0) and (time_reader == most_recent_time_graphed.tList[0]):
for i, param in enumerate(param_list, 1):
current_text = live_dict[param]
current_text.label.config(text=most_recent[0][i], fg="black", bg="white")
break #checks if the timestamp is exactly the same as prior, i.e. no new data points have been logged in this frame
#do I have to add an else?
else:
for i, key in enumerate(param_dict, 1):
current_plot = param_dict[key]
current_param_val = float(most_recent[0][i])
current_text = live_dict[key] #update to live text data summary
current_text.label.config(text=most_recent[0][i], fg="black", bg="white")
current_plot.plot_color = 'g'
data_stream = current_plot.incoming_data
time_stream = current_plot.tList
data_stream.insert(0, most_recent[0][i])
#time_f = datetime.strptime(most_recent[0][0], "%m/%d/%Y %H:%M:%S")
time_f = datetime.datetime.fromtimestamp(most_recent[0][0])
time_stream.insert(0, time_f)
if len(data_stream) < 20: #graph updates, growing to show 20 points
current_plot.make_plot()
else: #there are 20 points and more available, so animation occurs
data_stream.pop()
time_stream.pop()
current_plot.make_plot()
break
if time_reader.minute == 50 and most_recent_time_graphed.tList[1].minute == 49:
profile.disable()
result = io.StringIO()
pstats.Stats(profile,stream=result).print_stats()
result=result.getvalue()
# chop the string into a csv-like buffer
result='ncalls'+result.split('ncalls')[-1]
result='\n'.join([','.join(line.rstrip().split(None,5)) for line in result.split('\n')])
# save it to disk
with open('testing.csv', 'r') as f:
result = f.read() + 'hour: ' + str(time_reader.hour) + '\n' + result
with open('testing.csv', 'w+') as f:
f.write(result)
f.close()
else:
profile.disable()
#initialization
class AllWindow(tk.Tk):
def __init__(self, *args, **kwargs):
tk.Tk.__init__(self, *args, **kwargs)
#add title
tk.Tk.wm_title(self, "Matplotlib Live Plotting")
container = tk.Frame(self)
container.pack(side="top", fill="both", expand=True)
container.grid_rowconfigure(0, weight=1)
container.grid_columnconfigure(0, weight=1)
#show the frames
self.frames = {}
#remember to add page to this list when making new ones
frame = HomePage(container, self)
#set background color for the pages
frame.config(bg='white')
self.frames[HomePage] = frame
frame.grid(row=0, column=0, sticky="nsew")
self.show_frame(HomePage)
def show_frame(self, cont):
frame = self.frames[cont]
frame.tkraise()
#end program fcn triggered by quit button
def die(self):
exit()
#add home page
class HomePage(tk.Frame):
def __init__(self, parent, controller):
tk.Frame.__init__(self,parent)
canvas = FigureCanvasTkAgg(f, self)
#background = canvas.copy_from_bbox(f.bbox)
canvas.draw()
#embed graph into canvas
canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand = True)
#add navigation bar
toolbar = NavigationToolbar2Tk(canvas, self)
toolbar.update()
#data table labels
for i, param in enumerate(param_list): #tk.Label self refers to Homepage
param_label = tk.Label(self, text=param, fg="black", bg="white",
font = MEDIUM_FONT, borderwidth = 2, relief = "ridge",
width=16, height=1, anchor=W, justify=LEFT)
param_label.place(x=5, y=500+22*i)
for i, param in enumerate(param_list):
loading_text = tk.Label(self, text="Loading", fg="black", bg="white",
font = MEDIUM_FONT, borderwidth = 2, relief = "ridge",
width=7, height=1)
loading_text.place(x=140, y=500+22*i)
current_text = Live_Text(loading_text)
live_dict[param] = current_text
app = AllWindow()
#app.geometry('1280x623')
app.geometry('1917x970')
#update animation first
ani = animation.FuncAnimation(f, animate, interval=100)
#mainloop
app.mainloop()Actual outcome
The program eventually crashes after running for several days. Memory usage gradually increases throughout the program runtime. Additionally, profiling the animate function revealed that _invalidate_internal in Matplotlib seemed to be growing in its execution time. This is in line with #11956 and #11972, since _invalidate_internal is searching through transform._parents, which grows over time. However, these issues were resolved, so I am unsure why I am encountering the same problem.
Expected outcome
Expect the program to run continuously and indefinitely without memory usage increasing over time.
Operating system
Raspbian GNU/Linux 10 (buster)
Matplotlib Version
3.4.3
Matplotlib Backend
Qt5Agg
Python version
3.9
Jupyter version
No response
Other libraries
No response
Installation
pip
Conda channel
No response
Activity
[-][Bug]: [/-][+][Bug]: Memory leak from repeated live plotting[/+]QuLogic commentedon Nov 11, 2021
Where's the plot? This code has far too much extra stuff, and no Matplotlib in it.
bencaterine commentedon Nov 11, 2021
Added code to create a reproducible example.
jklymak commentedon Nov 11, 2021
@bencaterine Is it possible to minimize that and still reproduce the bug? If it is a memory leak in Matplotlib, it will hopefully be apparent with far less code.
bencaterine commentedon Nov 12, 2021
Yes - just updated with a minimized version
jklymak commentedon Nov 12, 2021
Please have a quick look at http://sscce.org for what we need. This is far too complicated to be actionable on our end.
tacaswell commentedon Nov 12, 2021
I definitly agree with @jklymak that there is still way too much here for us to debug, but from skimming the code I have a couple of big-picture comments:
animation.Animationis not doing much for you here. It really shines when you are using blitting (which you are not because you completely clear the axes (?) on ever update. You may be better off doing something likeand avoiding the animation module all together.
If you want to keep using the animation module, I would try passing
cache_frame_datato make sure we are not accidentally caching an infinite number of ints (the frame data falls back toitertools.count). I do not think we should be holding more than 100 of them by default, but if it takes days to leak, that is possible.It may also be worth trying to pass
repeat=False. We tend to assume that things going into the animation framework are finite and repeatable sequences (not open-ended data acquisition!)Another design aspect is that
FunctionAnimationis intended to have a more-or-less stateless function which gets passed the data to plot as input and then updates (in-place) a handful of artists. In this example you are doing a bunch of polling of (I assume) some hardware. It works, but is not how it was intended to be used (and you may be fighting the baked in assumptions). I would factor this asNotable this will also make testing easier as you can have
that so long as they
yieldthe same thing will be interchangeable!I may also be worth explicitly calling
gc.collect()occasionally, but if the leak is slow that means it is many small objects (not a few huge objects) so I suspect that Python's automatic sweeping is going to be happening often enough.It may be worth replacing
fill_betweenwithplotjust to go to a simpler plot and would help verify thatfill_betweenwas the problem).If you think particular objects are not getting gc'd at any point, a good trick is to grab a weakref to it and then register a finalizer that prints (if you never see the print that is what is "leaking!").
I am sympathetic to the difficulty of doing data acquisition as that is what I do for BNL (see https://blueskyproject.io) ;).
jklymak commentedon Jan 6, 2022
I'm going to close this for lack of response. Feel free to request a re-open if you still need help.