Persist gets into a bad state if kafka rolls

[ManApart]

Persist should recover from other parts of the system failing and then recovering. Rolling the pod resolved the issue, but should not be necessary.

AC

• When Kafka rolls, Persist should recover

18:48:53.545 [error] GenServer :brook_driver_kafka_persist_instance terminating
** (stop) [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
    (brod) src/brod_client.erl:554: :brod_client.ensure_metadata_connection/1
    (brod) src/brod_client.erl:300: :brod_client.handle_info/2
    (stdlib) gen_server.erl:637: :gen_server.try_dispatch/4
    (stdlib) gen_server.erl:711: :gen_server.handle_msg/6
    (stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: :init
18:48:53.546 [warn] Elixir.Elsa.DynamicProcessManager: initializer raised exception, retrying: {[{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}], {:gen_server, :call, [#PID<0.3785.0>, {:get_metadata, "event-stream"}, :infinity]}}
18:48:53.546 [error] GenServer #PID<0.3770.0> terminating
** (stop) [{{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
Last message: {:EXIT, #PID<0.3771.0>, [{{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]}
18:48:53.546 [error] GenServer #PID<0.3791.0> terminating
** (stop) exited in: :gen_server.call(:brook_driver_kafka_persist_instance, {:get_group_coordinator, "persist-event-stream"}, :infinity)
    ** (EXIT) [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
    (stdlib) gen_server.erl:223: :gen_server.call/3
    (brod) src/brod_client.erl:839: :brod_client.safe_gen_call/3
    (brod) src/brod_group_coordinator.erl:436: anonymous fn/2 in :brod_group_coordinator.discover_coordinator/1
    (brod) src/brod_group_coordinator.erl:436: :brod_group_coordinator.discover_coordinator/1
    (brod) src/brod_group_coordinator.erl:530: :brod_group_coordinator.do_stabilize/3
    (brod) src/brod_group_coordinator.erl:343: :brod_group_coordinator.handle_info/2
    (stdlib) gen_server.erl:637: :gen_server.try_dispatch/4
    (stdlib) gen_server.erl:711: :gen_server.handle_msg/6
Last message: {:lo_cmd_stabilize, 0, :undefined}
18:48:53.547 [error] GenServer #PID<0.3756.0> terminating
** (stop) exited in: GenServer.call({:via, Elsa.Registry, {:"elsa_registry_Elixir.Writer.Kafka.Topic_#PID<0.3727.0>", :producer_process_manager}}, :ready?, 10000)
    ** (EXIT) time out
Last message: {:EXIT, #PID<0.3727.0>, {:timeout, {GenServer, :call, [{:via, Elsa.Registry, {:"elsa_registry_Elixir.Writer.Kafka.Topic_#PID<0.3727.0>", :producer_process_manager}}, :ready?, 10000]}}}
18:48:53.551 [error] GenServer #PID<0.3779.0> terminating
** (stop) [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
Last message: {:EXIT, #PID<0.3780.0>, [{{:localhost, 9092}, {:econnrefused, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}, {{:"hindsight-hindsight-kafka-bootstrap", 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]}
18:48:53.572 [warn] Elixir.Elsa.DynamicProcessManager: initializer raised exception, retrying: %MatchError{term: {:error, :client_down}}