Chroniker stops working after 2-3 days and needs a restart.

[nayan576]

Please checkout the stack trace below. This is a very weird problem. Chroniker is running as a command under 'supervisor script'. It needs a process restart after every 2-3 days to keep working else it freezes and starts throwing the below error.

The database was completely operational at the time and used by other processes in normal fashion.

`
[2020-06-03 01:57:16] Running due jobs...

Exception in thread Thread-5007:
Traceback (most recent call last):
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 217, in ensure_connection
self.connect()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 195, in connect
self.connection = self.get_new_connection(conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 178, in get_new_connection
connection = Database.connect(**conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/hooks/database_dbapi2.py", line 101, in call
*args, **kwargs), self._nr_dbapi2_module, (args, kwargs))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/psycopg2/init.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: could not translate host name "xxxxxxxxx.rds.amazonaws.com" to address: System error

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/chroniker/management/commands/cronserver.py", line 19, in run
call_command('cron')
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/core/management/init.py", line 148, in call_command
return command.execute(*args, **defaults)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/core/management/base.py", line 364, in execute
output = self.handle(*args, **options)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/packages/wrapt/wrappers.py", line 508, in call
args, kwargs)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/api/function_trace.py", line 104, in literal_wrapper
return wrapped(*args, **kwargs)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/chroniker/management/commands/cron.py", line 338, in handle
kill_stalled_processes(dryrun=False)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/chroniker/management/commands/cron.py", line 34, in kill_stalled_processes
.values_list('current_pid', flat=True)))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/query.py", line 274, in iter
self._fetch_all()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/query.py", line 1242, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/query.py", line 182, in iter
for row in compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size):
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1085, in results_iter
results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1131, in execute_sql
cursor = self.connection.cursor()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 256, in cursor
return self._cursor()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 233, in _cursor
self.ensure_connection()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 217, in ensure_connection
self.connect()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/utils.py", line 89, in exit
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 217, in ensure_connection
self.connect()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 195, in connect
self.connection = self.get_new_connection(conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 178, in get_new_connection
connection = Database.connect(**conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/hooks/database_dbapi2.py", line 101, in call
*args, **kwargs), self._nr_dbapi2_module, (args, kwargs))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/psycopg2/init.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
django.db.utils.OperationalError: could not translate host name "xxxxxxxxxxxx.rds.amazonaws.com" to address: System error

Exception in thread Thread-5008:
Traceback (most recent call last):
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 217, in ensure_connection
self.connect()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 195, in connect
self.connection = self.get_new_connection(conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 178, in get_new_connection
connection = Database.connect(**conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/hooks/database_dbapi2.py", line 101, in call
*args, **kwargs), self._nr_dbapi2_module, (args, kwargs))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/psycopg2/init.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: could not translate host name "xxxxxxxxxxxxxxx.rds.amazonaws.com" to address: System error

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/chroniker/management/commands/cronserver.py", line 19, in run
call_command('cron')
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/core/management/init.py", line 148, in call_command
return command.execute(*args, **defaults)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/core/management/base.py", line 364, in execute
output = self.handle(*args, **options)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/packages/wrapt/wrappers.py", line 508, in call
args, kwargs)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/api/function_trace.py", line 104, in literal_wrapper
return wrapped(*args, **kwargs)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/chroniker/management/commands/cron.py", line 338, in handle
kill_stalled_processes(dryrun=False)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/chroniker/management/commands/cron.py", line 34, in kill_stalled_processes
.values_list('current_pid', flat=True)))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/query.py", line 274, in iter
self._fetch_all()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/query.py", line 1242, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/query.py", line 182, in iter
for row in compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size):
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1085, in results_iter
results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1131, in execute_sql
cursor = self.connection.cursor()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 256, in cursor
return self._cursor()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 233, in _cursor
self.ensure_connection()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 217, in ensure_connection
self.connect()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/utils.py", line 89, in exit
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 217, in ensure_connection
self.connect()
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/base/base.py", line 195, in connect
self.connection = self.get_new_connection(conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 178, in get_new_connection
connection = Database.connect(**conn_params)
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/newrelic/hooks/database_dbapi2.py", line 101, in call
*args, **kwargs), self._nr_dbapi2_module, (args, kwargs))
File "/home/ubuntu/.virtualenvs/C24Vajra/lib/python3.6/site-packages/psycopg2/init.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
django.db.utils.OperationalError: could not translate host name "xxxxxxxxxxxx.rds.amazonaws.com" to address: System error
`

[aaronn]

@nayan576 Does this still happen for you? Just want to confirm before adding Chroniker to our project.