Add some statefull property-based integration tests

src/mtp_handler.erl

@@ -26,9 +26,6 @@
 -define(MAX_UP_INIT_BUF_SIZE, 1024 * 1024).     %1mb
 
 -define(HEALTH_CHECK_INTERVAL, 5000).
--define(HEALTH_CHECK_MAX_QLEN, 300).
--define(HEALTH_CHECK_GC, 400 * 1024).            %400kb
--define(HEALTH_CHECK_MAX_MEM, 3 * 1024 * 1024).  %3mb
 
 -define(APP, mtproto_proxy).
 

src/mtproto_proxy_app.erl

@@ -16,9 +16,9 @@
 %%====================================================================
 start(_StartType, _StartArgs) ->
     Res = {ok, _} = mtproto_proxy_sup:start_link(),
-    io:format("+++++++++++++++++++++++++++++++++++++++~n"
-              "Erlang MTProto proxy by @seriyps https://github.com/seriyps/mtproto_proxy~n"
-              "Sponsored by and powers @socksy_bot~n"),
+    report("+++++++++++++++++++++++++++++++++++++++~n"
+           "Erlang MTProto proxy by @seriyps https://github.com/seriyps/mtproto_proxy~n"
+           "Sponsored by and powers @socksy_bot~n", []),
     [start_proxy(Where) || Where <- application:get_env(?APP, ports, [])],
     Res.
 
@@ -52,9 +52,13 @@ start_proxy(#{name := Name, port := Port, secret := Secret, tag := Tag} = P) ->
             "https://t.me/proxy?server=~s&port=~w&secret=~s",
             [application:get_env(?APP, external_ip, ListenIpStr),
              Port, Secret]),
-    io:format("Proxy started on ~s:~p with secret: ~s, tag: ~s~nUrl: ~s~n",
-              [ListenIpStr, Port, Secret, Tag, Url]),
+    report("Proxy started on ~s:~p with secret: ~s, tag: ~s~nUrl: ~s~n",
+           [ListenIpStr, Port, Secret, Tag, Url]),
     Res.
 
 stop_proxy(#{name := Name}) ->
     ranch:stop_listener(Name).
+
+report(Fmt, Args) ->
+    io:format(Fmt, Args),
+    lager:info(Fmt, Args).

test/mtp_prop_gen.erl

@@ -51,6 +51,4 @@ dc_id() ->
 
 codec() ->
     Protocols = [mtp_abridged, mtp_intermediate, mtp_secure],
-    proper_types:oneof(
-      [proper_types:exactly(P)
-       || P <- Protocols]).
+    proper_types:oneof(Protocols).

test/mtp_test_client.erl

@@ -48,7 +48,7 @@ recv_packet(#client{codec = Codec} = Client, Timeout) ->
 recv_packet_inner(#client{sock = Sock, codec = Codec0} = Client, Timeout) ->
     case gen_tcp:recv(Sock, 0, Timeout) of
         {ok, Stream} ->
-            io:format("~p: ~p~n", [byte_size(Stream), Stream]),
+            %% io:format("~p: ~p~n", [byte_size(Stream), Stream]),
             case mtp_codec:try_decode_packet(Stream, Codec0) of
                 {ok, Data, Codec} ->
                     {ok, Data, Client#client{codec = Codec}};

test/mtp_test_cmd_rpc.erl

@@ -30,11 +30,15 @@ handle_rpc({data, ConnId, Req}, St) ->
     case M:F(Opts, ConnId, St) of
         {reply, Resp, St1} ->
             {rpc, {proxy_ans, ConnId, term_to_packet(Resp)}, St1};
+        {close, St1} ->
+            {rpc, {close_ext, ConnId}, tombstone(ConnId, St1)};
         {return, What} ->
             What
     end;
 handle_rpc({remote_closed, ConnId}, St) ->
-    is_integer(maps:get(ConnId, St, undefined))
-        orelse error({unexpected_closed, ConnId}),
-    {noreply, St#{ConnId := tombstone}}.
+    {noreply, tombstone(ConnId, St)}.
 
+tombstone(ConnId, St) ->
+    ({ok, tombstone} =/= maps:find(ConnId, St))
+        orelse error({already_closed, ConnId}),
+    St#{ConnId => tombstone}.

test/mtp_test_datacenter.erl

@@ -8,6 +8,7 @@
          stop_dc/1,
          start_config_server/5,
          stop_config_server/1]).
+-export([middle_connections/1]).
 -export([dc_list_to_config/1]).
 -export([do/1]).
 
@@ -47,6 +48,10 @@ stop_dc(#{srv_ids := Ids} = Acc) ->
     ok = lists:foreach(fun mtp_test_middle_server:stop/1, Ids),
     {ok, maps:without([srv_ids], Acc1)}.
 
+middle_connections(#{srv_ids := Ids}) ->
+    lists:flatten([ranch:procs(Id, connections)
+                   || Id <- Ids]).
+
 %%
 %% Inets HTTPD to use as a mock for https://core.telegram.org
 %%

test/mtp_test_middle_server.erl

@@ -4,7 +4,8 @@
 -behaviour(gen_statem).
 
 -export([start/2,
-         stop/1]).
+         stop/1,
+         get_rpc_handler_state/1]).
 -export([start_link/4,
          ranch_init/1]).
 -export([init/1,
@@ -57,6 +58,9 @@ start(Id, #{port := _, secret := _} = Opts) ->
 stop(Id) ->
     ranch:stop_listener(Id).
 
+get_rpc_handler_state(Pid) ->
+    gen_statem:call(Pid, get_rpc_handler_state).
+
 %% Callbacks
 
 start_link(Ref, _, Transport, Opts) ->
@@ -155,6 +159,8 @@ on_tunnel(info, {tcp, _Sock, TcpData}, #t_state{codec = Codec0} = S) ->
                   {S2, S2#t_state.codec}
           end, S, TcpData, Codec0),
     {keep_state, activate(S2#t_state{codec = Codec1})};
+on_tunnel({call, From}, get_rpc_handler_state, #t_state{rpc_handler_state = HSt}) ->
+    {keep_state_and_data, [{reply, From, HSt}]};
 on_tunnel(Type, Event, S) ->
     handle_event(Type, Event, ?FUNCTION_NAME, S).
 

test/single_dc_SUITE.erl

@@ -13,6 +13,9 @@
          downstream_qlen_backpressure_case/1
         ]).
 
+-export([set_env/2,
+         reset_env/1]).
+
 -export([gen_rpc_replies/3]).
 
 -include_lib("common_test/include/ct.hrl").
@@ -49,7 +52,7 @@ end_per_testcase(Name, Cfg) ->
 
 %% @doc Send single packet and receive it back
 echo_secure_case({pre, Cfg}) ->
-    setup_single(?FUNCTION_NAME, ?LINE, #{}, Cfg);
+    setup_single(?FUNCTION_NAME, 10000 + ?LINE, #{}, Cfg);
 echo_secure_case({post, Cfg}) ->
     stop_single(Cfg);
 echo_secure_case(Cfg) when is_list(Cfg) ->
@@ -78,7 +81,7 @@ echo_secure_case(Cfg) when is_list(Cfg) ->
 
 %% @doc Send many packets and receive them back
 echo_abridged_many_packets_case({pre, Cfg}) ->
-    setup_single(?FUNCTION_NAME, ?LINE, #{}, Cfg);
+    setup_single(?FUNCTION_NAME, 10000 + ?LINE, #{}, Cfg);
 echo_abridged_many_packets_case({post, Cfg}) ->
     stop_single(Cfg);
 echo_abridged_many_packets_case(Cfg) when is_list(Cfg) ->
@@ -111,11 +114,11 @@ echo_abridged_many_packets_case(Cfg) when is_list(Cfg) ->
 
 %% @doc test downstream backpressure when size of non-acknowledged packets grows above threshold
 downstream_size_backpressure_case({pre, Cfg}) ->
-    Cfg1 = setup_single(?FUNCTION_NAME, ?LINE, #{rpc_handler => mtp_test_cmd_rpc}, Cfg),
+    Cfg1 = setup_single(?FUNCTION_NAME, 10000 + ?LINE, #{rpc_handler => mtp_test_cmd_rpc}, Cfg),
     %% Disable upstream healthchecks
-    application:set_env(?APP, upstream_healthchecks, []),
-    Cfg1;
+    set_env([{upstream_healthchecks, []}], Cfg1);
 downstream_size_backpressure_case({post, Cfg}) ->
+    reset_env(Cfg),
     stop_single(Cfg);
 downstream_size_backpressure_case(Cfg) when is_list(Cfg) ->
     DcId = ?config(dc_id, Cfg),
@@ -171,13 +174,13 @@ downstream_size_backpressure_case(Cfg) when is_list(Cfg) ->
 downstream_qlen_backpressure_case({pre, Cfg}) ->
     application:load(mtproto_proxy),
     %% Reducing downstream socket buffer size. Otherwise we can get queue overflow from just single
-    %% socket data packet
-    application:set_env(mtproto_proxy, downstream_socket_buffer_size, 1024),
-    Cfg1 = setup_single(?FUNCTION_NAME, ?LINE, #{rpc_handler => mtp_test_cmd_rpc}, Cfg),
+    %% socket data packet;
     %% Disable upstream healthchecks
-    application:set_env(?APP, upstream_healthchecks, []),
-    Cfg1;
+    Cfg1 = set_env([{downstream_socket_buffer_size, 1024},
+                    {upstream_healthchecks, []}], Cfg),
+    setup_single(?FUNCTION_NAME, 10000 + ?LINE, #{rpc_handler => mtp_test_cmd_rpc}, Cfg1);
 downstream_qlen_backpressure_case({post, Cfg}) ->
+    reset_env(Cfg),
     stop_single(Cfg);
 downstream_qlen_backpressure_case(Cfg) when is_list(Cfg) ->
     DcId = ?config(dc_id, Cfg),
@@ -222,13 +225,12 @@ gen_rpc_replies(#{packet := Packet, n := N}, ConnId, St) ->
 
 %% Helpers
 
-setup_single(Name, Offset, DcCfg0, Cfg) ->
+setup_single(Name, MtpPort, DcCfg0, Cfg) ->
     {ok, Pid} = mtp_test_metric:start_link(),
     PubKey = crypto:strong_rand_bytes(128),
     DcId = 1,
     Ip = {127, 0, 0, 1},
-    DcConf = [{DcId, Ip, 10000 + Offset}],
-    MtpPort = 10000 + Offset + 1,
+    DcConf = [{DcId, Ip, MtpPort + 10}],
     Secret = mtp_handler:hex(crypto:strong_rand_bytes(16)),
     Listeners = [#{name => Name,
                    port => MtpPort,
@@ -270,11 +272,11 @@ set_env(Env, Cfg) ->
          end || {K, V} <- Env],
     [{mtp_env, OldEnv} | Cfg].
 
-%% reset_env(Cfg) ->
-%%     OldEnv = ?config(mtp_env, Cfg),
-%%     [case V of
-%%          undefined ->
-%%              application:unset_env(mtproto_proxy, K);
-%%          {ok, Val} ->
-%%              application:set_env(mtproto_proxy, K, Val)
-%%      end || {K, V} <- OldEnv].
+reset_env(Cfg) ->
+    OldEnv = ?config(mtp_env, Cfg),
+    [case V of
+         undefined ->
+             application:unset_env(mtproto_proxy, K);
+         {ok, Val} ->
+             application:set_env(mtproto_proxy, K, Val)
+     end || {K, V} <- OldEnv].