Commit dfd60ef6 authored by Ralf's avatar Ralf
Browse files

send an email when updating the SpaceAI consistently fails

parent 97036270
......@@ -13,6 +13,7 @@ class SpaceApi:
self._remote_state = None
self._last_set_at = 0
self._running = True
self._fail_count = 0 # number of consecutive fails
self.set_state = ThreadFunction(self._set_state, "Space API")
waker.register(self.set_state, RETRY_TIME)
......@@ -44,7 +45,11 @@ class SpaceApi:
if self._local_state is not None and (self._local_state != self._remote_state or now > self._last_set_at+HEARTBEAT_TIME):
# take action!
success = self._do_request(self._local_state)
# TODO error too often -> log critical to send mails
if success:
self._remote_state = self._local_state
self._last_set_at = now
self._fail_count = 0
else:
self._fail_count += 1
if self._fail_count in (5, 50, 500):
logger.critical("Updating the SpaceAPI failed %d times in a row" % self._fail_count)
......@@ -43,7 +43,7 @@ except KeyboardInterrupt:
# this is what we waited for!
pass
logger.info("Terminating...")
logger.info("Terminating...") # somehow this does not arrive in the syslog
# bring 'em all down
the_waker.stop() # this one first, it "randomly" calls other threads
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment