summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTobias Henkel <tobias.henkel@bmw.de>2019-02-03 13:25:01 +0100
committerTobias Henkel <tobias.henkel@bmw.de>2019-02-03 17:16:44 +0100
commit3d9f47be1d315de3e2edfe789b2503a6cffe02a6 (patch)
tree0c17c042560554cc2294c6a2ee64518cd1d95ea1
parentb44b6c532ce9a999b135d909ae9f9caf0671784d (diff)
Properly close streams
When running test cases in the debugger it spits out resource warnings about non-closed streams [1]. Explicitly closing the streams of the subprocesses and log streamer fixes this warning. Maybe this even solves the memory leak we're currently seeing in the executors. [1] Trace: /zuul/executor/server.py:1894: ResourceWarning: unclosed file <_io.BufferedReader name=60> self.proc = None ResourceWarning: Enable tracemalloc to get the object allocation traceback /zuul/executor/server.py:124: ResourceWarning: unclosed file <_io.BufferedReader name=11> stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) ResourceWarning: Enable tracemalloc to get the object allocation traceback Change-Id: I65f191dc2e50f9c84f5bf6a3967d768d7ebe6b04
Notes
Notes (review): Code-Review+2: Monty Taylor <mordred@inaugust.com> Code-Review+2: Clint 'SpamapS' Byrum <clint@fewbar.com> Workflow+1: Clint 'SpamapS' Byrum <clint@fewbar.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 04 Feb 2019 18:51:29 +0000 Reviewed-on: https://review.openstack.org/634598 Project: openstack-infra/zuul Branch: refs/heads/master
-rw-r--r--tests/unit/test_streaming.py1
-rw-r--r--zuul/executor/server.py2
-rw-r--r--zuul/lib/log_streamer.py5
-rwxr-xr-xzuul/web/__init__.py2
4 files changed, 10 insertions, 0 deletions
diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py
index 2345aed..35fbcfa 100644
--- a/tests/unit/test_streaming.py
+++ b/tests/unit/test_streaming.py
@@ -55,6 +55,7 @@ class WSClient(WebSocketBaseClient):
55 self.send(json.dumps(req)) 55 self.send(json.dumps(req))
56 self.event.set() 56 self.event.set()
57 super(WSClient, self).run() 57 super(WSClient, self).run()
58 self.close()
58 59
59 60
60class TestLogStreamer(tests.base.BaseTestCase): 61class TestLogStreamer(tests.base.BaseTestCase):
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 219c234..7b4dfe2 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -141,6 +141,7 @@ class DiskAccountant(object):
141 .format(size=size, job=dirname, limit=self.limit)) 141 .format(size=size, job=dirname, limit=self.limit))
142 self.usage_func(dirname, size) 142 self.usage_func(dirname, size)
143 du.wait() 143 du.wait()
144 du.stdout.close()
144 after = time.time() 145 after = time.time()
145 # Sleep half as long as that took, or 1s, whichever is longer 146 # Sleep half as long as that took, or 1s, whichever is longer
146 delay_time = max((after - before) / 2, 1.0) 147 delay_time = max((after - before) / 2, 1.0)
@@ -1891,6 +1892,7 @@ class AnsibleJob(object):
1891 self.log.debug("Stopped disk job killer") 1892 self.log.debug("Stopped disk job killer")
1892 1893
1893 with self.proc_lock: 1894 with self.proc_lock:
1895 self.proc.stdout.close()
1894 self.proc = None 1896 self.proc = None
1895 1897
1896 if timeout and watchdog.timed_out: 1898 if timeout and watchdog.timed_out:
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index 1eed621..16b7222 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -98,6 +98,11 @@ class RequestHandler(streamer_utils.BaseFingerRequestHandler):
98 if self.follow_log(log): 98 if self.follow_log(log):
99 break 99 break
100 else: 100 else:
101 if log is not None:
102 try:
103 log.file.close()
104 except Exception:
105 pass
101 return 106 return
102 107
103 def chunk_log(self, log_file): 108 def chunk_log(self, log_file):
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index dbbb707..3acf480 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -185,9 +185,11 @@ class LogStreamer(object):
185 if data: 185 if data:
186 self.websocket.send(data, False) 186 self.websocket.send(data, False)
187 self.zuulweb.stream_manager.unregisterStreamer(self) 187 self.zuulweb.stream_manager.unregisterStreamer(self)
188 self.finger_socket.close()
188 return self.websocket.logClose(1000, "No more data") 189 return self.websocket.logClose(1000, "No more data")
189 else: 190 else:
190 self.zuulweb.stream_manager.unregisterStreamer(self) 191 self.zuulweb.stream_manager.unregisterStreamer(self)
192 self.finger_socket.close()
191 return self.websocket.logClose(1000, "Remote error") 193 return self.websocket.logClose(1000, "Remote error")
192 194
193 195