Add option to disable crc32 check in mtp_full to save CPU

README.md

@@ -415,6 +415,8 @@ mtp_policy_table:add(customer_domains, tls_domain, "my-client1.example.com").'
 ```
 
 And then use http://seriyps.ru/mtpgen.html to generate unique link for them.
+Be aware that domains table will be reset if proxy is restarted! Make sure you re-add them
+when proxy restarts (eg, via [systemd hook script](https://unix.stackexchange.com/q/326181/70382)).
 
 ### IPv6
 
@@ -478,6 +480,12 @@ it will use less CPU and will be better protected from replay attacks, but will
     max_age_minutes => 1440}},
 ```
 
+One more option to decrease CPU usage is to disable CRC32 checksum check:
+
+```erlang
+{mtp_full_check_crc32, false},
+```
+
 Also, for highload setups it's recommended to increase sysctl parameters:
 
 ```

src/mtp_down_conn.erl

@@ -495,10 +495,11 @@ down_handshake1(S) ->
     Schema = 1,                                 %AES
     Msg = mtp_rpc:encode_nonce({nonce, KeySelector, Schema, CryptoTs, Nonce}),
     Deadline = erlang:send_after(?HANDSHAKE_TIMEOUT, self(), handshake_timeout),
+    CheckCRC = application:get_env(?APP, mtp_full_check_crc32, true),
     S1 = S#state{stage = handshake_1,
                  %% Use fake encryption codec
                  codec = mtp_codec:new(mtp_noop_codec, mtp_noop_codec:new(),
-                                       mtp_full, mtp_full:new(-2, -2),
+                                       mtp_full, mtp_full:new(-2, -2, CheckCRC),
                                        false, undefined, ?MAX_CODEC_BUFFERS),
                  stage_state = {Deadline, KeySelector, Nonce, CryptoTs, Key}},
     down_send(Msg, S1).

src/mtp_full.erl

@@ -11,7 +11,7 @@
 -module(mtp_full).
 -behaviour(mtp_codec).
 
--export([new/0, new/2,
+-export([new/0, new/3,
          try_decode_packet/2,
          encode_packet/2]).
 -export_type([codec/0]).
@@ -20,7 +20,9 @@
 
 -record(full_st,
         {enc_seq_no :: integer(),
-         dec_seq_no :: integer()}).
+         dec_seq_no :: integer(),
+         check_crc = true :: boolean()}).
+
 -define(MIN_MSG_LEN, 12).
 -define(MAX_MSG_LEN,  16777216).                %2^24 - 16mb
 
@@ -32,17 +34,18 @@
 
 
 new() ->
-    new(0, 0).
+    new(0, 0, true).
 
-new(EncSeqNo, DecSeqNo) ->
+new(EncSeqNo, DecSeqNo, CheckCRC) ->
     #full_st{enc_seq_no = EncSeqNo,
-             dec_seq_no = DecSeqNo}.
+             dec_seq_no = DecSeqNo,
+             check_crc = CheckCRC}.
 
 try_decode_packet(<<4:32/little, Tail/binary>>, S) ->
     %% Skip padding
     try_decode_packet(Tail, S);
 try_decode_packet(<<Len:32/little, PktSeqNo:32/signed-little, Tail/binary>>,
-                  #full_st{dec_seq_no = SeqNo} = S) ->
+                  #full_st{dec_seq_no = SeqNo, check_crc = CheckCRC} = S) ->
     ((Len rem byte_size(?PAD)) == 0)
         orelse error({wrong_alignement, Len}),
     ((?MIN_MSG_LEN =< Len) and (Len =< ?MAX_MSG_LEN))
@@ -52,9 +55,14 @@ try_decode_packet(<<Len:32/little, PktSeqNo:32/signed-little, Tail/binary>>,
     BodyLen = Len - 4 - 4 - 4,
     case Tail of
         <<Body:BodyLen/binary, CRC:32/little, Rest/binary>> ->
-            PacketCrc = erlang:crc32([<<Len:32/little, PktSeqNo:32/little>> | Body]),
-            (CRC == PacketCrc)
-                orelse error({wrong_checksum, CRC, PacketCrc}),
+            case CheckCRC of
+                true ->
+                    PacketCrc = erlang:crc32([<<Len:32/little, PktSeqNo:32/little>> | Body]),
+                    (CRC == PacketCrc)
+                        orelse error({wrong_checksum, CRC, PacketCrc});
+                false ->
+                    ok
+            end,
             %% TODO: predict padding size from padding_size(Len)
             {ok, Body, trim_padding(Rest), S#full_st{dec_seq_no = SeqNo + 1}};
         _ ->

src/mtproto_proxy.app.src

@@ -130,6 +130,10 @@
         %% {upstream_socket_buffer_size, 51200},   %50kb
         %% {downstream_socket_buffer_size, 512000},   %500kb
 
+        %% Whether we should check CRC32 sum of packets encoded with mtp_full
+        %% codec. Setting this to `false' decreases CPU usage. Default: true
+        %% {mtp_full_check_crc32, true},
+
         %% Where to fetch telegram proxy configuration
         %% Mostly used to testing
         %% {proxy_secret_url, "https://core.telegram.org/getProxySecret"},

test/bench_codec_decode.erl

@@ -11,6 +11,7 @@
          intermediate/1, bench_intermediate/2,
          secure/1, bench_secure/2,
          full/1, bench_full/2,
+         full_nocheck/1, bench_full_nocheck/2,
          aes_cbc/1, bench_aes_cbc/2,
          obfuscated/1, bench_obfuscated/2,
          fold_dd_codec/1, bench_fold_dd_codec/2,
@@ -86,6 +87,17 @@ decode_all_full(Stream0, Codec0) ->
     end.
 
 
+%% @doc bench mtp_full with disabled CRC32 verification check
+full_nocheck(init) ->
+    mtp_full:new(0, 0, false);
+full_nocheck({input, Codec}) ->
+    Packets = mk_front_packets(),
+    encode_all(Packets, mtp_full, Codec).
+
+bench_full_nocheck(Stream, Codec) ->
+    decode_all_full(Stream, Codec).
+
+
 %% @doc bench aes_cbc decryption
 aes_cbc(init) ->
     Key = binary:copy(<<0>>, 32),

test/mtp_test_middle_server.erl

@@ -74,7 +74,7 @@ ranch_init({Ref, Transport, Opts}) ->
 
 init({Socket, Transport, Opts}) ->
     Codec = mtp_codec:new(mtp_noop_codec, mtp_noop_codec:new(),
-                          mtp_full, mtp_full:new(-2, -2)),
+                          mtp_full, mtp_full:new(-2, -2, true)),
     State = #hs_state{sock = Socket,
                       transport = Transport,
                       secret = maps:get(secret, Opts),

test/prop_mtp_codec.erl

@@ -145,7 +145,7 @@ fullcbc_stream(Key, Iv, Stream) ->
 
 mk_fullcbc_codec(EncKey, EncIv, DecKey, DecIv) ->
     Crypto = mtp_aes_cbc:new(EncKey, EncIv, DecKey, DecIv, 16),
-    Packet = mtp_full:new(1, 1),
+    Packet = mtp_full:new(1, 1, true),
     mtp_codec:new(mtp_aes_cbc, Crypto,
                   mtp_full, Packet).
 

test/prop_mtp_full.erl

@@ -9,10 +9,11 @@ prop_codec(doc) ->
     "Tests that any 4-byte aligned binary can be encoded and decoded back".
 
 prop_codec() ->
-    ?FORALL(Bin, mtp_prop_gen:packet_4b(), codec(Bin)).
+    ?FORALL({CheckCRC, Bin}, {proper_types:boolean(), mtp_prop_gen:packet_4b()},
+            codec(Bin, CheckCRC)).
 
-codec(Bin) ->
-    Codec = mtp_full:new(),
+codec(Bin, CheckCRC) ->
+    Codec = mtp_full:new(0, 0, CheckCRC),
     {Data, Codec1} = mtp_full:encode_packet(Bin, Codec),
     {ok, Decoded, <<>>, _} = mtp_full:try_decode_packet(iolist_to_binary(Data), Codec1),
     Decoded == Bin.