Skip to content

Commit

Permalink
Keep track of connections, introduce per-vhost limits
Browse files Browse the repository at this point in the history 
Fixes #500, #627.

Squashed commit of the following:

commit 88036dc
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Thu Jul 21 03:31:25 2016 +0300

    Refactor

commit fc84b7a
Merge: df745e2 df28c63
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 20 18:30:19 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit df745e2
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 20 18:04:59 2016 +0300

    Force close connections when vhost is deleted

    Fixes #627, related to #500.

commit 2167f8f
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 20 16:00:35 2016 +0300

    Add tests for per-vhost connection limits

commit 2a032a3
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 20 01:53:07 2016 +0300

    Rename a few tests

commit 86ce592
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 20 01:44:10 2016 +0300

    Tests for connection re-registration idempotency

commit a774c7b
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Tue Jul 19 04:05:20 2016 +0300

    Ask nodes that come back to re-register their connections

    Depending on the partition handling mode used there may or may not
    be any clients still connected. We make sure that registration
    and deregistration functions are idempotent and assume there
    may be connections on the node that has come back.

    Point of improvement: when a node comes back up, N-1 nodes
    will tell it to re-register connections. It could be fewer
    than N-1, ideally just 1.

commit 24e4c0e
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Mon Jul 18 17:05:17 2016 +0300

    Fix boot step

commit 62da3c6
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Mon Jul 18 11:16:21 2016 +0300

    Compile

commit b656f9e
Merge: f2831e1 492406e
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Thu Jul 14 15:25:49 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit f2831e1
Merge: e5858e9 7b10a4e
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Thu Jul 7 13:45:31 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit e5858e9
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 6 12:32:56 2016 +0300

    Towards working connection re-registration after (inter-node) network splits

commit 548df73
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jul 6 12:32:07 2016 +0300

    Make network split simulation work as expected

commit 4028c66
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Tue Jul 5 14:43:37 2016 +0300

    Close connections using rabbit_ct_client_helpers

    Per discussion with @dumbbell.

commit 26fecc9
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Tue Jul 5 04:17:52 2016 +0300

    Extract connection limit partition tests into a separate suite

commit 8a466f1
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Tue Jul 5 04:17:41 2016 +0300

    Better logging

commit b06de9b
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Mon Jul 4 02:54:54 2016 +0300

    Modify a test so that it (expectedly) fails

commit 078a78a
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Mon Jul 4 02:44:58 2016 +0300

    Towards covering node termination/unavailability in connection tracking

commit ab99361
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sun Jul 3 15:25:10 2016 +0300

    These are moved to rabbit_ct_broker_helpers

commit 520b6ef
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sun Jul 3 03:54:52 2016 +0300

    {allow,block}_traffic_between/2 are moved to rabbit_ct_broker_helpers

commit b842eaa
Merge: 26eb1fa d4f031e
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sun Jul 3 03:14:27 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit 26eb1fa
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sun Jul 3 02:39:09 2016 +0300

    dist_proxy helpers moved to rabbit_ct_broker_helpers

commit 3d741f4
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sun Jul 3 01:28:44 2016 +0300

    Cluster node shutdown test

commit 57c7129
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sat Jul 2 23:01:46 2016 +0300

    Refactor

commit b736b30
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sat Jul 2 22:49:42 2016 +0300

    More tests

commit dc1cb5f
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sat Jul 2 22:27:16 2016 +0300

    More tests

commit e94edfe
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sat Jul 2 17:08:34 2016 +0300

    Initial per-vhost connection limit tests

commit 15b7b4e
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sat Jul 2 15:04:57 2016 +0300

    Adapt to master, compile

commit dc7f333
Merge: e4884ff bb1fa55
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Sat Jul 2 02:44:18 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit e4884ff
Merge: 71e2710 f0f43f8
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Wed Jun 29 14:27:40 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit 71e2710
Merge: b1ec9f3 704a2b5
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Thu Mar 31 01:55:29 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

    Conflicts:
    	src/rabbit_control_main.erl
    	src/rabbit_types.erl

commit b1ec9f3
Author: Michael Klishin <mklishin@pivotal.io>
Date:   Mon Feb 15 13:51:37 2016 +0300

    Stub out event handlers for #627 and #628

commit f3cfb57
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Sat Feb 13 01:33:50 2016 +0300

    Use a counter column to track number of connections per vhost

    Limit query time is now 50-70 microseconds for
    50M connections.

commit e9132f1
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Fri Feb 12 06:23:51 2016 +0300

    Ignore ./debug

commit 976e3ae
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Fri Feb 12 06:20:01 2016 +0300

    Switch to ets:select_count/2

commit ec23cf1
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Thu Feb 11 05:11:08 2016 +0300

    Enforce max connection limit

    Also introduce `rabbitmqctl clear_vhost_limits`
    and fix rabbitmqctl(1).

commit ba20887
Merge: 49a1886 8974581
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Thu Feb 11 02:16:24 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit 49a1886
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Wed Feb 10 16:45:00 2016 +0300

    Spelling

commit 723e6e4
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Wed Feb 10 16:31:34 2016 +0300

    Create secondary indices on rabbit_tracked_connection.vhost and username

commit b468c0f
Merge: 6940d05 0120438
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Wed Feb 10 12:23:38 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit 6940d05
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Mon Feb 8 01:30:03 2016 +0300

    Spam

commit 032c2a6
Merge: 46da39c 2374ae8
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Fri Feb 5 23:48:38 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit 46da39c
Merge: 655e351 05361e6
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Wed Feb 3 11:20:08 2016 +0300

    Merge branch 'master' into rabbitmq-server-500

commit 655e351
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Wed Feb 3 11:19:23 2016 +0300

    Store and delete tracked connections in a table

commit 4e849cf
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Tue Jan 19 17:56:14 2016 +0300

    Compile

commit 504adde
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Tue Jan 19 17:55:08 2016 +0300

    Switch to a handler for connection tracking (WIP)

commit 3e1d2b4
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Tue Jan 19 14:46:09 2016 +0300

    Migrations for virtual host limits and tracked connections

commit 7499020
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Fri Jan 8 19:40:36 2016 +0300

    Compile

commit f3a1101
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Fri Jan 8 19:14:12 2016 +0300

    Switch rabbitmqctl set_vhost_limits to use JSON payload values

    Just like policies do.

commit 7fc5f1a
Author: Michael Klishin <michael@clojurewerkz.org>
Date:   Wed Jan 6 19:07:50 2016 +0300

    Stub out set_vhost_limits in ctl
  • Loading branch information
@michaelklishin
michaelklishin committed Jul 22, 2016
1 parent 6ad34a4 commit 9adb6cd
Show file tree
Hide file tree
Showing 15 changed files with 1,526 additions and 11 deletions.
55 changes: 52 additions & 3 deletions docs/rabbitmqctl.1.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1214,6 +1214,55 @@
</variablelist>
</refsect2>

<refsect2>
<title>Virtual Host Limits</title>
<para>
It is possible to enforce certain limits on virtual hosts.
</para>
<variablelist>
<varlistentry>
<term><cmdsynopsis><command>set_vhost_limits</command> <arg choice="opt">-p <replaceable>vhostpath</replaceable></arg> <arg choice="req"><replaceable>definition</replaceable></arg></cmdsynopsis></term>
<listitem>
<para>
Sets virtual host limits
</para>
<variablelist>
<varlistentry>
<term>definition</term>
<listitem><para>
The definition of the limits, as a
JSON term. In most shells you are very likely to
need to quote this.

Recognised limits: max-connections (0 means "no limit").
</para></listitem>
</varlistentry>
</variablelist>
<para role="example-prefix">For example:</para>
<screen role="example">rabbitmqctl set_vhost_limits -p qa_env '{"max-connections": 1024}'</screen>
<para role="example">
This command limits the max number of concurrent connections in vhost <command>qa_env</command>
to 1024.
</para>
</listitem>
</varlistentry>

<varlistentry>
<term><cmdsynopsis><command>clear_vhost_limits</command> <arg choice="opt">-p <replaceable>vhostpath</replaceable></arg></cmdsynopsis></term>
<listitem>
<para>
Clears virtual host limits
</para>
<para role="example-prefix">For example:</para>
<screen role="example">rabbitmqctl clear_vhost_limits -p qa_env</screen>
<para role="example">
This command clears vhost limits in vhost <command>qa_env</command>.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect2>

<refsect2>
<title>Server Status</title>
<para>
Expand Down Expand Up @@ -2036,9 +2085,9 @@
<varlistentry>
<term>fraction</term>
<listitem><para>
Limit relative to the total amount available RAM
as a non-negative floating point number.
Values lower than 1.0 can be dangerous and
Limit relative to the total amount available RAM
as a non-negative floating point number.
Values lower than 1.0 can be dangerous and
should be used carefully.
</para></listitem>
</varlistentry>
Expand Down
1 change: 0 additions & 1 deletion include/rabbit_cli.hrl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
-define(OFFLINE_OPT, "--offline").
-define(ONLINE_OPT, "--online").


-define(NODE_DEF(Node), {?NODE_OPT, {option, Node}}).
-define(QUIET_DEF, {?QUIET_OPT, flag}).
-define(VHOST_DEF, {?VHOST_OPT, {option, "/"}}).
Expand Down
5 changes: 5 additions & 0 deletions src/rabbit.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,11 @@
{mfa, {rabbit_direct, boot, []}},
{requires, log_relay}]}).

-rabbit_boot_step({connection_tracker,
[{description, "helps track node-local connections"},
{mfa, {rabbit_connection_tracker, boot, []}},
{requires, log_relay}]}).

-rabbit_boot_step({networking,
[{mfa, {rabbit_networking, boot, []}},
{requires, log_relay}]}).
Expand Down
99 changes: 99 additions & 0 deletions src/rabbit_connection_tracker.erl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
%% The contents of this file are subject to the Mozilla Public License
%% Version 1.1 (the "License"); you may not use this file except in
%% compliance with the License. You may obtain a copy of the License
%% at http://www.mozilla.org/MPL/
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and
%% limitations under the License.
%%
%% The Original Code is RabbitMQ.
%%
%% The Initial Developer of the Original Code is GoPivotal, Inc.
%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved.
%%

-module(rabbit_connection_tracker).

%% Abstracts away how tracked connection records are stored
%% and queried.
%%
%% See also:
%%
%% * rabbit_connection_tracking_handler
%% * rabbit_reader
%% * rabbit_event

-behaviour(gen_server2).

%% API
-export([boot/0, start_link/0, reregister/1]).

%% gen_fsm callbacks
-export([init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3]).

-define(SERVER, ?MODULE).


%%%===================================================================
%%% API
%%%===================================================================

boot() ->
{ok, _} = start_link(),
ok.

start_link() ->
gen_server2:start_link({local, ?SERVER}, ?MODULE, [], []).

reregister(Node) ->
rabbit_log:info("Telling node ~p to re-register tracked connections", [Node]),
gen_server2:cast({?SERVER, Node}, reregister).

%%%===================================================================
%%% gen_server callbacks
%%%===================================================================

init([]) ->
{ok, {}}.

handle_call(_Req, _From, State) ->
{noreply, State}.

handle_cast(reregister, State) ->
Cs = rabbit_networking:connections_local(),
rabbit_log:info("Connection tracker: asked to re-register ~p client connections", [length(Cs)]),
case Cs of
[] -> ok;
Cs ->
[reregister_connection(C) || C <- Cs],
ok
end,
rabbit_log:info("Done re-registering client connections"),
{noreply, State}.

handle_info(_Req, State) ->
{noreply, State}.

terminate(_Reason, _State) ->
ok.

code_change(_OldVsn, State, _Extra) ->
{ok, State}.

%%%===================================================================
%%% Internal functions
%%%===================================================================

reregister_connection(Conn) ->
try
Conn ! reregister
catch _:Error ->
rabbit_log:error("Failed to re-register connection ~p after a network split: ~p", [Conn, Error])
end.
Loading

0 comments on commit 9adb6cd

Please sign in to comment.