From c74cd00bf94279b757d34e527e2c2fb7070857ba Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Mon, 8 Feb 2016 21:32:04 +0300 Subject: [PATCH 01/45] Prevent infinite loop on 'queue.declare' `rabbit_channel:handle_method` for `queue.declare` could go into infinite loop, when it tries do define a queue which got somehow corrupted. This fix at least will help with symptoms of this corruption - channel will quickly become unstuck. Loop happens due to somewhat special handling of `not_found` in 'queue.declare' - it is treated not as an error here, but as a permission to proceed with queue creation. As the base case of recursion in `rabbit_amqqueue:with/4` is actuallly about some unrecoverable error, it makes sense to always return `absent` error here. It will still result in `not_found` error for clients, but internally it'll be handled correctly. --- src/rabbit_amqqueue.erl | 6 ++++-- src/rabbit_misc.erl | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index 1bcf69b9..23d40b02 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -413,10 +413,12 @@ not_found_or_absent_dirty(Name) -> with(Name, F, E) -> with(Name, F, E, 2000). -with(Name, _F, E, 0) -> - E(not_found_or_absent_dirty(Name)); with(Name, F, E, RetriesLeft) -> case lookup(Name) of + {ok, Q = #amqqueue{}} when RetriesLeft =:= 0 -> + %% Something bad happened to that queue, we are bailing out + %% on processing current request. + E({absent, Q, timeout}); {ok, Q = #amqqueue{state = crashed}} -> E({absent, Q, crashed}); {ok, Q = #amqqueue{pid = QPid}} -> diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index 231db347..14645605 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -310,7 +310,11 @@ absent(#amqqueue{name = QueueName, pid = QPid, durable = true}, nodedown) -> absent(#amqqueue{name = QueueName}, crashed) -> protocol_error(not_found, - "~s has crashed and failed to restart", [rs(QueueName)]). + "~s has crashed and failed to restart", [rs(QueueName)]); + +absent(#amqqueue{name = QueueName}, timeout) -> + protocol_error(not_found, + "failed to perform operation on ~s due to timeout", [rs(QueueName)]). type_class(byte) -> int; type_class(short) -> int; From b5c84387e80a64b220414c7d58fb8162719c69f0 Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Fri, 12 Feb 2016 17:19:32 +0300 Subject: [PATCH 02/45] Remove custom stderr formatting Opening several ports for single fd is considered undefined behaviour in erlang. It's safe to replace this whole function with `io:format` when erlang 17 or later is used. Because writing to standard_error with io:format is synchronous - after this call has returned data was definitely sent to the port. And `erlang:halt/` guarantees that this data will be flushed afterwards. --- src/rabbit_misc.erl | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index b7441375..35e5c105 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -655,18 +655,7 @@ format_many(List) -> lists:flatten([io_lib:format(F ++ "~n", A) || {F, A} <- List]). format_stderr(Fmt, Args) -> - case os:type() of - {unix, _} -> - Port = open_port({fd, 0, 2}, [out]), - port_command(Port, io_lib:format(Fmt, Args)), - port_close(Port); - {win32, _} -> - %% stderr on Windows is buffered and I can't figure out a - %% way to trigger a fflush(stderr) in Erlang. So rather - %% than risk losing output we write to stdout instead, - %% which appears to be unbuffered. - io:format(Fmt, Args) - end, + io:format(standard_error, Fmt, Args), ok. unfold(Fun, Init) -> From fecd0e5ae5e866ce443c164cd4b0c1dbc0d566e4 Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Fri, 12 Feb 2016 17:19:32 +0300 Subject: [PATCH 03/45] Remove custom stderr formatting Opening several ports for single fd is considered undefined behaviour in erlang. It's safe to replace this whole function with `io:format`. Because writing to standard_error with io:format is synchronous - after this call has returned data was definitely sent to the port. And `erlang:halt/` guarantees that this data will be flushed afterwards. Tested on windows/linux with R16B03 and 18.X. Closes #53 for `stable` --- src/rabbit_misc.erl | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index b7441375..35e5c105 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -655,18 +655,7 @@ format_many(List) -> lists:flatten([io_lib:format(F ++ "~n", A) || {F, A} <- List]). format_stderr(Fmt, Args) -> - case os:type() of - {unix, _} -> - Port = open_port({fd, 0, 2}, [out]), - port_command(Port, io_lib:format(Fmt, Args)), - port_close(Port); - {win32, _} -> - %% stderr on Windows is buffered and I can't figure out a - %% way to trigger a fflush(stderr) in Erlang. So rather - %% than risk losing output we write to stdout instead, - %% which appears to be unbuffered. - io:format(Fmt, Args) - end, + io:format(standard_error, Fmt, Args), ok. unfold(Fun, Init) -> From 2a24cb53f1d427914d05fdb0aca2be497d522c55 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Tue, 2 Feb 2016 19:05:19 +0000 Subject: [PATCH 04/45] Adds caching of channel_operation_timeout in process dictionary. Acquisition of channel_operation_timeout from dictionary on rabbit_amqqueue:notify_down_all/3 call. References: rabbitmq/rabbitmq-server-248 --- src/rabbit_channel.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 79a2d1c4..c67d227e 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -392,6 +392,7 @@ init([Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, User, VHost, rabbit_event:notify(channel_created, infos(?CREATION_EVENT_KEYS, State2)), rabbit_event:if_enabled(State2, #ch.stats_timer, fun() -> emit_stats(State2) end), + put(channel_termination_timeout, ?CHANNEL_OPERATION_TIMEOUT), {ok, State2, hibernate, {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. @@ -1770,7 +1771,9 @@ notify_queues(State = #ch{consumer_mapping = Consumers, delivering_queues = DQ }) -> QPids = sets:to_list( sets:union(sets:from_list(consumer_queues(Consumers)), DQ)), - {rabbit_amqqueue:notify_down_all(QPids, self()), State#ch{state = closing}}. + {rabbit_amqqueue:notify_down_all(QPids, self(), + get(channel_termination_timeout)), + State#ch{state = closing}}. foreach_per_queue(_F, []) -> ok; From a1fee052b0b2c8cc99dbaa10c160dc8913164178 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Tue, 2 Feb 2016 19:16:45 +0000 Subject: [PATCH 05/45] Adds CHANNEL_OPERATION_TIMEOUT macro. References: rabbitmq/rabbitmq-server-248 --- include/rabbit.hrl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/rabbit.hrl b/include/rabbit.hrl index dfbf2222..40193be1 100644 --- a/include/rabbit.hrl +++ b/include/rabbit.hrl @@ -159,6 +159,8 @@ -define(EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, <<"exchange-delete-in-progress">>). +-define(CHANNEL_OPERATION_TIMEOUT, rabbit_misc:get_channel_operation_timeout()). + %% Trying to send a term across a cluster larger than 2^31 bytes will %% cause the VM to exit with "Absurdly large distribution output data %% buffer". So we limit the max message size to 2^31 - 10^6 bytes (1MB From cf9a8b5093774f88fc2f2dd50d61e95b0b38aa99 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Wed, 17 Feb 2016 15:01:17 +0000 Subject: [PATCH 06/45] Adds notify_down_all/3 clause to handle timeouts. Ref: rabbitmq/rabbitmq-server#248 --- src/rabbit_amqqueue.erl | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index 1bcf69b9..cd8d19c7 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -31,7 +31,7 @@ -export([consumers/1, consumers_all/1, consumers_all/3, consumer_info_keys/0]). -export([basic_get/4, basic_consume/10, basic_cancel/4, notify_decorators/1]). -export([notify_sent/2, notify_sent_queue_down/1, resume/2]). --export([notify_down_all/2, activate_limit_all/2, credit/5]). +-export([notify_down_all/2, notify_down_all/3, activate_limit_all/2, credit/5]). -export([on_node_up/1, on_node_down/1]). -export([update/2, store_queue/1, update_decorators/1, policy_changed/2]). -export([start_mirroring/1, stop_mirroring/1, sync_mirrors/1, @@ -160,6 +160,8 @@ -spec(ack/3 :: (pid(), [msg_id()], pid()) -> 'ok'). -spec(reject/4 :: (pid(), [msg_id()], boolean(), pid()) -> 'ok'). -spec(notify_down_all/2 :: (qpids(), pid()) -> ok_or_errors()). +-spec(notify_down_all/3 :: (qpids(), pid(), non_neg_integer()) + -> ok_or_errors()). -spec(activate_limit_all/2 :: (qpids(), pid()) -> ok_or_errors()). -spec(basic_get/4 :: (rabbit_types:amqqueue(), pid(), boolean(), pid()) -> {'ok', non_neg_integer(), qmsg()} | 'empty'). @@ -687,13 +689,23 @@ reject(QPid, Requeue, MsgIds, ChPid) -> delegate:cast(QPid, {reject, Requeue, MsgIds, ChPid}). notify_down_all(QPids, ChPid) -> - {_, Bads} = delegate:call(QPids, {notify_down, ChPid}), - case lists:filter( - fun ({_Pid, {exit, {R, _}, _}}) -> rabbit_misc:is_abnormal_exit(R); - ({_Pid, _}) -> false - end, Bads) of - [] -> ok; - Bads1 -> {error, Bads1} + notify_down_all(QPids, ChPid, ?CHANNEL_OPERATION_TIMEOUT). + +notify_down_all(QPids, ChPid, Timeout) -> + case rpc:call(node(), delegate, call, + [QPids, {notify_down, ChPid}], Timeout) of + {badrpc, timeout} -> {error, {channel_operation_timeout, Timeout}}; + {badrpc, Reason} -> {error, Reason}; + {_, Bads} -> + case lists:filter( + fun ({_Pid, {exit, {R, _}, _}}) -> + rabbit_misc:is_abnormal_exit(R); + ({_Pid, _}) -> false + end, Bads) of + [] -> ok; + Bads1 -> {error, Bads1} + end; + Error -> {error, Error} end. activate_limit_all(QPids, ChPid) -> From 9fcc0ae3d79c32cdfc4fa2c6ffa62def0e119c31 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Wed, 17 Feb 2016 15:13:11 +0000 Subject: [PATCH 07/45] Adds get_channel_operation_timeout/0. Ref: rabbitmq/rabbitmq-server#248 --- src/rabbit_misc.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index 231db347..8fbdfa68 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -71,6 +71,7 @@ -export([store_proc_name/1, store_proc_name/2]). -export([moving_average/4]). -export([get_env/3]). +-export([get_channel_operation_timeout/0]). -export([random/1]). %% Horrible macro to use in guards @@ -260,7 +261,7 @@ -spec(moving_average/4 :: (float(), float(), float(), float() | 'undefined') -> float()). -spec(get_env/3 :: (atom(), atom(), term()) -> term()). - +-spec(get_channel_operation_timeout/0 :: () -> non_neg_integer()). -spec(random/1 :: (non_neg_integer()) -> non_neg_integer()). -endif. @@ -1118,6 +1119,13 @@ get_env(Application, Key, Def) -> undefined -> Def end. +get_channel_operation_timeout() -> + %% Default channel_operation_timeout set to net_ticktime + 10s to + %% give allowance for any down messages to be received first, + %% whenever it is used for cross-node calls with timeouts. + Default = (net_kernel:get_net_ticktime() + 10) * 1000, + application:get_env(rabbit, channel_operation_timeout, Default). + moving_average(_Time, _HalfLife, Next, undefined) -> Next; %% We want the Weight to decrease as Time goes up (since Weight is the From 4ec5d6795710725bca0f74614a99cf4f0b9ccf3e Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Wed, 17 Feb 2016 17:54:25 +0000 Subject: [PATCH 08/45] Renames CHANNEL_OPERATION_TIMEOUT dict key. --- src/rabbit_channel.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index c67d227e..ded782a0 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -392,7 +392,7 @@ init([Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, User, VHost, rabbit_event:notify(channel_created, infos(?CREATION_EVENT_KEYS, State2)), rabbit_event:if_enabled(State2, #ch.stats_timer, fun() -> emit_stats(State2) end), - put(channel_termination_timeout, ?CHANNEL_OPERATION_TIMEOUT), + put(channel_operation_timeout, ?CHANNEL_OPERATION_TIMEOUT), {ok, State2, hibernate, {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. @@ -1772,7 +1772,7 @@ notify_queues(State = #ch{consumer_mapping = Consumers, QPids = sets:to_list( sets:union(sets:from_list(consumer_queues(Consumers)), DQ)), {rabbit_amqqueue:notify_down_all(QPids, self(), - get(channel_termination_timeout)), + get(channel_operation_timeout)), State#ch{state = closing}}. foreach_per_queue(_F, []) -> From 9cb793179622b75ccb51080ac74df5a17a74d8a7 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Thu, 18 Feb 2016 16:06:17 +0000 Subject: [PATCH 09/45] Moves channel state update from notify_queues/1 to channel.close handle clause, following discussion with Michael Klishin. Ref: rabbitmq/rabbitmq-server#248 --- src/rabbit_channel.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index ded782a0..f6a9b4c3 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -884,7 +884,7 @@ handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> - {ok, State1} = notify_queues(State), + {_Res, State} = notify_queues(State), %% We issue the channel.close_ok response after a handshake with %% the reader, the other half of which is ready_for_close. That %% way the reader forgets about the channel before we send the @@ -895,7 +895,7 @@ handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> %% the termination and hence be sent to the old, now dead/dying %% channel process, instead of a new process, and thus lost. ReaderPid ! {channel_closing, self()}, - {noreply, State1}; + {noreply, State#ch{state = closing}}; %% Even though the spec prohibits the client from sending commands %% while waiting for the reply to a synchronous command, we generally @@ -1773,7 +1773,7 @@ notify_queues(State = #ch{consumer_mapping = Consumers, sets:union(sets:from_list(consumer_queues(Consumers)), DQ)), {rabbit_amqqueue:notify_down_all(QPids, self(), get(channel_operation_timeout)), - State#ch{state = closing}}. + State}. foreach_per_queue(_F, []) -> ok; From 327e923106ac96c38c3a2663ed8f38bca9d93bd4 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Fri, 19 Feb 2016 05:50:39 +0300 Subject: [PATCH 10/45] Set state to closing in notify_queues notify_queues is used in multiple call sites, so keep it compatible. --- src/rabbit_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index f6a9b4c3..3e48ea58 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1773,7 +1773,7 @@ notify_queues(State = #ch{consumer_mapping = Consumers, sets:union(sets:from_list(consumer_queues(Consumers)), DQ)), {rabbit_amqqueue:notify_down_all(QPids, self(), get(channel_operation_timeout)), - State}. + State#ch{state = closing}}. foreach_per_queue(_F, []) -> ok; From c8ee7f2b94837d82ec90ebba5171bd1b7283047b Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Fri, 19 Feb 2016 06:42:51 +0300 Subject: [PATCH 11/45] Revert "Set state to closing in notify_queues" This reverts commit 327e923106ac96c38c3a2663ed8f38bca9d93bd4. --- src/rabbit_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 3e48ea58..f6a9b4c3 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -1773,7 +1773,7 @@ notify_queues(State = #ch{consumer_mapping = Consumers, sets:union(sets:from_list(consumer_queues(Consumers)), DQ)), {rabbit_amqqueue:notify_down_all(QPids, self(), get(channel_operation_timeout)), - State#ch{state = closing}}. + State}. foreach_per_queue(_F, []) -> ok; From 02752f1c4c0b3f9226c30ad2811e01d24f481a36 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Fri, 19 Feb 2016 06:58:36 +0300 Subject: [PATCH 12/45] Revert "Moves channel state update from notify_queues/1 to channel.close" This reverts commit 9cb793179622b75ccb51080ac74df5a17a74d8a7. notify_queues is used in multiple call sites, lets revert this to ensure backwards compatibility. --- src/rabbit_channel.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index f6a9b4c3..ded782a0 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -884,7 +884,7 @@ handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> - {_Res, State} = notify_queues(State), + {ok, State1} = notify_queues(State), %% We issue the channel.close_ok response after a handshake with %% the reader, the other half of which is ready_for_close. That %% way the reader forgets about the channel before we send the @@ -895,7 +895,7 @@ handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> %% the termination and hence be sent to the old, now dead/dying %% channel process, instead of a new process, and thus lost. ReaderPid ! {channel_closing, self()}, - {noreply, State#ch{state = closing}}; + {noreply, State1}; %% Even though the spec prohibits the client from sending commands %% while waiting for the reply to a synchronous command, we generally @@ -1773,7 +1773,7 @@ notify_queues(State = #ch{consumer_mapping = Consumers, sets:union(sets:from_list(consumer_queues(Consumers)), DQ)), {rabbit_amqqueue:notify_down_all(QPids, self(), get(channel_operation_timeout)), - State}. + State#ch{state = closing}}. foreach_per_queue(_F, []) -> ok; From 38e1e3d681166d461c9e373eaf48209fb1440176 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Mon, 22 Feb 2016 16:30:36 +0000 Subject: [PATCH 13/45] Ignore result from notify_queues/1. Ref: rabbitmq/rabbitmq-server#248 --- src/rabbit_channel.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index ded782a0..13520d93 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -884,7 +884,7 @@ handle_method(_Method, _, State = #ch{state = closing}) -> {noreply, State}; handle_method(#'channel.close'{}, _, State = #ch{reader_pid = ReaderPid}) -> - {ok, State1} = notify_queues(State), + {_Result, State1} = notify_queues(State), %% We issue the channel.close_ok response after a handshake with %% the reader, the other half of which is ready_for_close. That %% way the reader forgets about the channel before we send the From 78999f5f38835268ef300be575cc524d0a4b83c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Wed, 24 Feb 2016 13:14:44 +0100 Subject: [PATCH 14/45] format_hard_error/1: Format Erlang term if it's not a string --- src/rabbit_reader.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 118bea24..73513f9a 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -787,7 +787,10 @@ termination_kind(_) -> uncontrolled. format_hard_error(#amqp_error{name = N, explanation = E, method = M}) -> io_lib:format("operation ~s caused a connection exception ~s: ~p", [M, N, E]); format_hard_error(Reason) -> - Reason. + case io_lib:deep_char_list(Reason) of + true -> Reason; + false -> rabbit_misc:format("~p", [Reason]) + end. log_hard_error(#v1{connection_state = CS, connection = #connection{ From 7bd33a1eb41d8a53be38af8395a5e6229d135440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Wed, 24 Feb 2016 13:16:30 +0100 Subject: [PATCH 15/45] log_hard_error/3: Prepend lines with ' ' after the first This improves the readability with Lager. --- src/rabbit_reader.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index be8d37f2..fce3f27d 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -799,7 +799,7 @@ log_hard_error(#v1{connection_state = CS, vhost = VHost}}, Channel, Reason) -> rabbit_connection:error( "Error on AMQP connection ~p (~s, vhost: '~s'," - " user: '~s', state: ~p), channel ~p:~n~s~n", + " user: '~s', state: ~p), channel ~p:~n ~s~n", [self(), ConnName, VHost, User#user.username, CS, Channel, format_hard_error(Reason)]). handle_exception(State = #v1{connection_state = closed}, Channel, Reason) -> From a827ee7c84f743f24ae2429e996350434b8c0d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Thu, 25 Feb 2016 14:31:38 +0100 Subject: [PATCH 16/45] Travis CI: Test with Erlang 18.2 instead of 18.0 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 668c0ff8..b6676a0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: otp_release: - "R16B03-1" - "17.5" - - "18.0" + - "18.2" # The checkout made by Travis is a "detached HEAD". We switch back # to a tag or a branch. This pleases our git_rmq fetch method in From bc55ea564b22fbffee3bf845db6bbb17a2766f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Thu, 25 Feb 2016 14:31:57 +0100 Subject: [PATCH 17/45] Travis CI: Fix how we recreate the tested tag/branch This should fix "build" failures for pull requests. --- .travis.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b6676a0d..bd8ecaac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,15 @@ otp_release: # to a tag or a branch. This pleases our git_rmq fetch method in # rabbitmq-components.mk and the proper tag/branch is selected in # dependencies too. -before_script: (test "$TRAVIS_TAG" && git checkout "$TRAVIS_TAG") || (test "$TRAVIS_BRANCH" && git checkout -b "$TRAVIS_BRANCH") +# +# FIXME: There is still one problem: for builds triggered by a pull +# request, $TRAVIS_BRANCH contains the target branch name, not the +# soruce branch name. Therefore, we can't rely on automatic checkout +# of corresponding branches in dependencies. For instance, if the pull +# request comes from a branch "rabbitmq-server-123", based on "stable", +# then this command will checkout "stable" and we won't try to checkout +# "rabbitmq-server-123" in dependencies. +before_script: git checkout -B "${TRAVIS_TAG:-${TRAVIS_BRANCH}}" script: make tests From da7529cbee789f36835162220697e55c1a6f5dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Thu, 25 Feb 2016 15:34:33 +0100 Subject: [PATCH 18/45] Travis CI: Make sure a "master" branch exists ... event a fake one. --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bd8ecaac..9986cece 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,13 @@ otp_release: # request comes from a branch "rabbitmq-server-123", based on "stable", # then this command will checkout "stable" and we won't try to checkout # "rabbitmq-server-123" in dependencies. -before_script: git checkout -B "${TRAVIS_TAG:-${TRAVIS_BRANCH}}" +# +# We also make sure the "master" branch exists, because +# rabbitmq-components.mk expects it. If it's missing, we just create a +# fake branch pointing to the same commit as $TRAVIS_BRANCH. +before_script: + - git checkout -B "${TRAVIS_TAG:-${TRAVIS_BRANCH}}" + - git rev-parse --verify -q master -- || git branch master script: make tests From f861346c2801c9f14c1bc7267183dbd16448b518 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Thu, 7 Jan 2016 19:20:16 +0100 Subject: [PATCH 19/45] Version support for time_compat References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 184 ++++++++++++++++++++++++++++ src/time_compat.erl | 277 +++++++++++++++++++++++++++---------------- 2 files changed, 362 insertions(+), 99 deletions(-) create mode 100644 src/code_version.erl diff --git a/src/code_version.erl b/src/code_version.erl new file mode 100644 index 00000000..0a19a942 --- /dev/null +++ b/src/code_version.erl @@ -0,0 +1,184 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ Federation. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% +-module(code_version). + +-export([update/1]). + +update(Module) -> + AbsCode = get_abs_code(Module), + Forms = replace_forms(Module, get_otp_version() >= 18, AbsCode), + Code = compile_forms(Forms), + load_code(Module, Code). + +load_code(Module, Code) -> + unload(Module), + case code:load_binary(Module, "loaded by rabbit_common", Code) of + {module, _} -> + ok; + {error, _Reason} -> + throw(cannot_load) + end. + +unload(Module) -> + code:soft_purge(Module), + code:delete(Module). + +compile_forms(Forms) -> + case compile:forms(Forms, [debug_info]) of + {ok, _ModName, Code} -> + Code; + {ok, _ModName, Code, _Warnings} -> + Code; + _ -> + throw(cannot_compile_forms) + end. + +get_abs_code(Module) -> + get_forms(get_object_code(Module)). + +get_object_code(Module) -> + case code:get_object_code(Module) of + {_Mod, Code, _File} -> + Code; + error -> + throw(not_found) + end. + +get_forms(Code) -> + case beam_lib:chunks(Code, [abstract_code]) of + {ok, {_, [{abstract_code, {raw_abstract_v1, Forms}}]}} -> + Forms; + {ok, {_, [{abstract_code, no_abstract_code}]}} -> + throw(no_abstract_code); + {error, beam_lib, Reason} -> + throw({no_abstract_code, Reason}) + end. + +get_otp_version() -> + Version = erlang:system_info(otp_release), + case re:run(Version, "^[0-9][0-9]", [{capture, first, list}]) of + {match, [V]} -> + list_to_integer(V); + _ -> + %% Could be anything below R17, we are not interested + 0 + end. + +get_original_pairs(VersionSupport) -> + [{Orig, Arity} || {Orig, _Pre, _Post, Arity} <- VersionSupport]. + +get_delete_pairs(true, VersionSupport) -> + [{Pre, Arity} || {_Orig, Pre, _Post, Arity} <- VersionSupport]; +get_delete_pairs(false, VersionSupport) -> + [{Post, Arity} || {_Orig, _Pre, Post, Arity} <- VersionSupport]. + +get_rename_pairs(true, VersionSupport) -> + [{Post, Arity} || {_Orig, _Pre, Post, Arity} <- VersionSupport]; +get_rename_pairs(false, VersionSupport) -> + [{Pre, Arity} || {_Orig, Pre, _Post, Arity} <- VersionSupport]. + +get_name_pairs(true, VersionSupport) -> + [{Post, Orig} || {Orig, _Pre, Post, _Arity} <- VersionSupport]; +get_name_pairs(false, VersionSupport) -> + [{Pre, Orig} || {Orig, Pre, _Post, _Arity} <- VersionSupport]. + +delete_abstract_functions(ToDelete) -> + fun(Tree, Function) -> + case lists:member(Function, ToDelete) of + true -> + erl_syntax:comment(["Deleted unused function"]); + false -> + Tree + end + end. + +rename_abstract_functions(ToRename, ToName) -> + fun(Tree, {Name, _Arity} = Function) -> + case lists:member(Function, ToRename) of + true -> + FunctionName = proplists:get_value(Name, ToName), + erl_syntax:function( + erl_syntax:atom(FunctionName), + erl_syntax:function_clauses(Tree)); + false -> + Tree + end + end. + +replace_forms(Module, IsPost18, AbsCode) -> + Attr = Module:module_info(attributes), + VersionSupport = proplists:get_value(version_support, Attr), + Original = get_original_pairs(VersionSupport), + ToDelete = get_delete_pairs(IsPost18, VersionSupport), + DeleteFun = delete_abstract_functions(ToDelete ++ Original), + AbsCode0 = replace_function_forms(AbsCode, DeleteFun), + ToRename = get_rename_pairs(IsPost18, VersionSupport), + ToName = get_name_pairs(IsPost18, VersionSupport), + RenameFun = rename_abstract_functions(ToRename, ToName), + remove_exports(replace_function_forms(AbsCode0, RenameFun), ToDelete ++ ToRename). + +replace_function_forms(AbsCode, Fun) -> + ReplaceFunction = + fun(Tree) -> + case erl_syntax_lib:analyze_function(Tree) of + {_N, _A} = Function -> + Fun(Tree, Function); + _Other -> Tree + end + end, + Filter = fun(Tree) -> + case erl_syntax:type(Tree) of + function -> ReplaceFunction(Tree); + _Other -> Tree + end + end, + fold_syntax_tree(Filter, AbsCode). + +filter_export_pairs(Info, ToDelete) -> + lists:filter(fun(Pair) -> + not lists:member(Pair, ToDelete) + end, Info). + +remove_exports(AbsCode, ToDelete) -> + RemoveExports = + fun(Tree) -> + case erl_syntax_lib:analyze_attribute(Tree) of + {export, Info} -> + Remaining = filter_export_pairs(Info, ToDelete), + rebuild_export(Remaining); + _Other -> Tree + end + end, + Filter = fun(Tree) -> + case erl_syntax:type(Tree) of + attribute -> RemoveExports(Tree); + _Other -> Tree + end + end, + fold_syntax_tree(Filter, AbsCode). + +rebuild_export(Args) -> + erl_syntax:attribute( + erl_syntax:atom(export), + [erl_syntax:list( + [erl_syntax:arity_qualifier(erl_syntax:atom(N), + erl_syntax:integer(A)) + || {N, A} <- Args])]). + +fold_syntax_tree(Filter, Forms) -> + Tree = erl_syntax:form_list(Forms), + NewTree = erl_syntax_lib:map(Filter, Tree), + erl_syntax:revert_forms(NewTree). diff --git a/src/time_compat.erl b/src/time_compat.erl index b87c6cc5..934099e7 100644 --- a/src/time_compat.erl +++ b/src/time_compat.erl @@ -43,160 +43,240 @@ %% where it has not yet been deprecated. %% +-version_support( + [{monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 0}, + {monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 1}, + {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 0}, + {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 1}, + {os_system_time, os_system_time_pre_18, os_system_time_post_18, 0}, + {os_system_time, os_system_time_pre_18, os_system_time_post_18, 1}, + {time_offset, time_offset_pre_18, time_offset_post_18, 0}, + {time_offset, time_offset_pre_18, time_offset_post_18, 1}, + {convert_time_unit, convert_time_unit_pre_18, convert_time_unit_post_18, 0}, + {timestamp, timestamp_pre_18, timestamp_post_18, 0}, + {unique_integer, unique_integer_pre_18, unique_integer_post_18, 0}, + {unique_integer, unique_integer_pre_18, unique_integer_post_18, 1}]). + -export([monotonic_time/0, - monotonic_time/1, - erlang_system_time/0, - erlang_system_time/1, - os_system_time/0, - os_system_time/1, - time_offset/0, - time_offset/1, - convert_time_unit/3, - timestamp/0, - unique_integer/0, - unique_integer/1, - monitor/2, - system_info/1, - system_flag/2]). + monotonic_time_pre_18/0, + monotonic_time_post_18/0, + monotonic_time/1, + monotonic_time_pre_18/1, + monotonic_time_post_18/1, + erlang_system_time/0, + erlang_system_time_pre_18/0, + erlang_system_time_post_18/0, + erlang_system_time/1, + erlang_system_time_pre_18/1, + erlang_system_time_post_18/1, + os_system_time/0, + os_system_time_pre_18/0, + os_system_time_post_18/0, + os_system_time/1, + os_system_time_pre_18/1, + os_system_time_post_18/1, + time_offset/0, + time_offset_pre_18/0, + time_offset_post_18/0, + time_offset/1, + time_offset_pre_18/1, + time_offset_post_18/1, + convert_time_unit/3, + convert_time_unit_pre_18/3, + convert_time_unit_post_18/3, + timestamp/0, + timestamp_pre_18/0, + timestamp_post_18/0, + unique_integer/0, + unique_integer_pre_18/0, + unique_integer_post_18/0, + unique_integer/1, + unique_integer_pre_18/1, + unique_integer_post_18/1, + monitor/2, + system_info/1, + system_flag/2]). monotonic_time() -> + code_version:update(?MODULE), + time_compat:monotonic_time(). + +monotonic_time_post_18() -> + erlang:monotonic_time(). + +monotonic_time_pre_18() -> + erlang_system_time_fallback(). + +monotonic_time(Unit) -> + code_version:update(?MODULE), + time_compat:monotonic_time(Unit). + +monotonic_time_post_18(Unit) -> try - erlang:monotonic_time() + erlang:monotonic_time(Unit) catch - error:undef -> - %% Use Erlang system time as monotonic time - erlang_system_time_fallback() + error:badarg -> + erlang:error(badarg, [Unit]) end. -monotonic_time(Unit) -> +monotonic_time_pre_18(Unit) -> + %% Use Erlang system time as monotonic time + STime = erlang_system_time_fallback(), try - erlang:monotonic_time(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]); - error:undef -> - %% Use Erlang system time as monotonic time - STime = erlang_system_time_fallback(), - try convert_time_unit_fallback(STime, native, Unit) - catch + catch error:bad_time_unit -> erlang:error(badarg, [Unit]) - end end. erlang_system_time() -> + code_version:update(?MODULE), + time_compat:erlang_system_time(). + +erlang_system_time_post_18() -> + erlang:system_time(). + +erlang_system_time_pre_18() -> + erlang_system_time_fallback(). + +erlang_system_time(Unit) -> + code_version:update(?MODULE), + time_compat:erlang_system_time(Unit). + +erlang_system_time_post_18(Unit) -> try - erlang:system_time() + erlang:system_time(Unit) catch - error:undef -> - erlang_system_time_fallback() + error:badarg -> + erlang:error(badarg, [Unit]) end. -erlang_system_time(Unit) -> +erlang_system_time_pre_18(Unit) -> + STime = erlang_system_time_fallback(), try - erlang:system_time(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]); - error:undef -> - STime = erlang_system_time_fallback(), - try convert_time_unit_fallback(STime, native, Unit) - catch + catch error:bad_time_unit -> erlang:error(badarg, [Unit]) - end end. os_system_time() -> + code_version:update(?MODULE), + time_compat:os_system_time(). + +os_system_time_post_18() -> + os:system_time(). + +os_system_time_pre_18() -> + os_system_time_fallback(). + +os_system_time(Unit) -> + code_version:update(?MODULE), + time_compat:os_system_time(Unit). + +os_system_time_post_18(Unit) -> try - os:system_time() + os:system_time(Unit) catch - error:undef -> - os_system_time_fallback() + error:badarg -> + erlang:error(badarg, [Unit]) end. -os_system_time(Unit) -> +os_system_time_pre_18(Unit) -> + STime = os_system_time_fallback(), try - os:system_time(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]); - error:undef -> - STime = os_system_time_fallback(), - try convert_time_unit_fallback(STime, native, Unit) - catch + catch error:bad_time_unit -> erlang:error(badarg, [Unit]) - end end. time_offset() -> + code_version:update(?MODULE), + time_compat:time_offset(). + +time_offset_post_18() -> + erlang:time_offset(). + +time_offset_pre_18() -> + %% Erlang system time and Erlang monotonic + %% time are always aligned + 0. + +time_offset(Unit) -> + code_version:update(?MODULE), + time_compat:time_offset(Unit). + +time_offset_post_18(Unit) -> try - erlang:time_offset() + erlang:time_offset(Unit) catch - error:undef -> - %% Erlang system time and Erlang monotonic - %% time are always aligned - 0 + error:badarg -> + erlang:error(badarg, [Unit]) end. -time_offset(Unit) -> +time_offset_pre_18(Unit) -> try - erlang:time_offset(Unit) + _ = integer_time_unit(Unit) catch - error:badarg -> - erlang:error(badarg, [Unit]); - error:undef -> - try - _ = integer_time_unit(Unit) - catch error:bad_time_unit -> erlang:error(badarg, [Unit]) - end, - %% Erlang system time and Erlang monotonic - %% time are always aligned - 0 - end. + end, + %% Erlang system time and Erlang monotonic + %% time are always aligned + 0. convert_time_unit(Time, FromUnit, ToUnit) -> + code_version:update(?MODULE), + time_compat:convert_time_unit(Time, FromUnit, ToUnit). + +convert_time_unit_post_18(Time, FromUnit, ToUnit) -> try - erlang:convert_time_unit(Time, FromUnit, ToUnit) + erlang:convert_time_unit(Time, FromUnit, ToUnit) catch - error:undef -> - try - convert_time_unit_fallback(Time, FromUnit, ToUnit) - catch - _:_ -> - erlang:error(badarg, [Time, FromUnit, ToUnit]) - end; - error:Error -> + error:Error -> erlang:error(Error, [Time, FromUnit, ToUnit]) end. -timestamp() -> +convert_time_unit_pre_18(Time, FromUnit, ToUnit) -> try - erlang:timestamp() + convert_time_unit_fallback(Time, FromUnit, ToUnit) catch - error:undef -> - erlang:now() + _:_ -> + erlang:error(badarg, [Time, FromUnit, ToUnit]) end. +timestamp() -> + code_version:update(?MODULE), + time_compat:timestamp(). + +timestamp_post_18() -> + erlang:timestamp(). + +timestamp_pre_18() -> + erlang:now(). + unique_integer() -> - try - erlang:unique_integer() - catch - error:undef -> - {MS, S, US} = erlang:now(), - (MS*1000000+S)*1000000+US - end. + code_version:update(?MODULE), + time_compat:unique_integer(). + +unique_integer_post_18() -> + erlang:unique_integer(). + +unique_integer_pre_18() -> + {MS, S, US} = erlang:now(), + (MS*1000000+S)*1000000+US. unique_integer(Modifiers) -> + code_version:update(?MODULE), + time_compat:unique_integer(Modifiers). + +unique_integer_post_18(Modifiers) -> try - erlang:unique_integer(Modifiers) + erlang:unique_integer(Modifiers) catch - error:badarg -> - erlang:error(badarg, [Modifiers]); - error:undef -> - case is_valid_modifier_list(Modifiers) of + error:badarg -> + erlang:error(badarg, [Modifiers]) + end. + +unique_integer_pre_18(Modifiers) -> + case is_valid_modifier_list(Modifiers) of true -> %% now() converted to an integer %% fullfill the requirements of @@ -206,7 +286,6 @@ unique_integer(Modifiers) -> (MS*1000000+S)*1000000+US; false -> erlang:error(badarg, [Modifiers]) - end end. monitor(Type, Item) -> From 133c65a1c9e3e2934b49c8f69684deaf31da2ed8 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Fri, 8 Jan 2016 15:22:31 +0100 Subject: [PATCH 20/45] Documentation and error control References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 69 +++++++++++++++++++++++++++++++++++++++----- src/time_compat.erl | 2 ++ 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/code_version.erl b/src/code_version.erl index 0a19a942..9d2ed8f5 100644 --- a/src/code_version.erl +++ b/src/code_version.erl @@ -17,19 +17,59 @@ -export([update/1]). +%%---------------------------------------------------------------------------- +%% API +%%---------------------------------------------------------------------------- + +%%---------------------------------------------------------------------------- +%% @doc Reads the abstract code of the given `Module`, modifies it to adapt to +%% the current Erlang version, compiles and loads the result. +%% This function finds the current Erlang version and then selects the function +%% call for that version, removing all other versions declared in the original +%% beam file. `code_version:update/1` is triggered by the module itself the +%% first time an affected function is called. +%% +%% The purpose of this functionality is to support the new time API introduced +%% in ERTS 7.0, while providing compatibility with previous versions. +%% +%% `Module` must contain an attribute `version_support` containing a list of +%% tuples: {OriginalFuntion, PreErlang18Function, PostErlang18Function, Arity} +%% +%% All these new functions may be exported, and implemented as follows: +%% +%% OriginalFunction() -> +%% code_version:update(?MODULE), +%% ?MODULE:OriginalFunction(). +%% +%% PostErlang18Function() -> +%% %% implementation using new time API +%% .. +%% +%% PreErlang18Function() -> +%% %% implementation using fallback solution +%% .. +%% +%% See `time_compat.erl` for an example. +%% +%% end +%%---------------------------------------------------------------------------- +-spec update(atom()) -> ok | no_return(). update(Module) -> AbsCode = get_abs_code(Module), Forms = replace_forms(Module, get_otp_version() >= 18, AbsCode), Code = compile_forms(Forms), load_code(Module, Code). +%%---------------------------------------------------------------------------- +%% Internal functions +%%---------------------------------------------------------------------------- load_code(Module, Code) -> unload(Module), case code:load_binary(Module, "loaded by rabbit_common", Code) of {module, _} -> ok; - {error, _Reason} -> - throw(cannot_load) + {error, Reason} -> + throw({cannot_load, Module, Reason}) end. unload(Module) -> @@ -42,8 +82,8 @@ compile_forms(Forms) -> Code; {ok, _ModName, Code, _Warnings} -> Code; - _ -> - throw(cannot_compile_forms) + Error -> + throw({cannot_compile_forms, Error}) end. get_abs_code(Module) -> @@ -54,15 +94,15 @@ get_object_code(Module) -> {_Mod, Code, _File} -> Code; error -> - throw(not_found) + throw({not_found, Module}) end. get_forms(Code) -> case beam_lib:chunks(Code, [abstract_code]) of {ok, {_, [{abstract_code, {raw_abstract_v1, Forms}}]}} -> Forms; - {ok, {_, [{abstract_code, no_abstract_code}]}} -> - throw(no_abstract_code); + {ok, {Module, [{abstract_code, no_abstract_code}]}} -> + throw({no_abstract_code, Module}); {error, beam_lib, Reason} -> throw({no_abstract_code, Reason}) end. @@ -90,6 +130,7 @@ get_rename_pairs(true, VersionSupport) -> get_rename_pairs(false, VersionSupport) -> [{Pre, Arity} || {_Orig, Pre, _Post, Arity} <- VersionSupport]. +%% Pairs of {Renamed, OriginalName} functions get_name_pairs(true, VersionSupport) -> [{Post, Orig} || {Orig, _Pre, Post, _Arity} <- VersionSupport]; get_name_pairs(false, VersionSupport) -> @@ -119,16 +160,28 @@ rename_abstract_functions(ToRename, ToName) -> end. replace_forms(Module, IsPost18, AbsCode) -> + %% Obtain attribute containing the list of functions that must be updated Attr = Module:module_info(attributes), VersionSupport = proplists:get_value(version_support, Attr), + %% Get pairs of {Function, Arity} for the triggering functions, which + %% are also the final function names. Original = get_original_pairs(VersionSupport), + %% Get pairs of {Function, Arity} for the unused version ToDelete = get_delete_pairs(IsPost18, VersionSupport), + %% Delete original functions (those that trigger the code update) and + %% the unused version ones DeleteFun = delete_abstract_functions(ToDelete ++ Original), AbsCode0 = replace_function_forms(AbsCode, DeleteFun), + %% Get pairs of {Function, Arity} for the current version which must be + %% renamed ToRename = get_rename_pairs(IsPost18, VersionSupport), + %% Get paris of {Renamed, OriginalName} functions ToName = get_name_pairs(IsPost18, VersionSupport), + %% Rename versioned functions with their final name RenameFun = rename_abstract_functions(ToRename, ToName), - remove_exports(replace_function_forms(AbsCode0, RenameFun), ToDelete ++ ToRename). + %% Remove exports of all versioned functions + remove_exports(replace_function_forms(AbsCode0, RenameFun), + ToDelete ++ ToRename). replace_function_forms(AbsCode, Fun) -> ReplaceFunction = diff --git a/src/time_compat.erl b/src/time_compat.erl index 934099e7..2d91ccb9 100644 --- a/src/time_compat.erl +++ b/src/time_compat.erl @@ -43,6 +43,8 @@ %% where it has not yet been deprecated. %% +%% Declare versioned functions to allow dynamic code loading, +%% depending on the Erlang version running. See 'code_version.erl' for details -version_support( [{monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 0}, {monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 1}, From 5f8314ae3a2ab71c61471b1048713aa0f3935ea3 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Fri, 8 Jan 2016 15:54:21 +0100 Subject: [PATCH 21/45] Version support for ssl_compat References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/ssl_compat.erl | 78 ++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/src/ssl_compat.erl b/src/ssl_compat.erl index d9cf3901..2c787011 100644 --- a/src/ssl_compat.erl +++ b/src/ssl_compat.erl @@ -20,44 +20,60 @@ %% this module. -compile(nowarn_deprecated_function). +%% Declare versioned functions to allow dynamic code loading, +%% depending on the Erlang version running. See 'code_version.erl' for details +-version_support( + [{connection_information, connection_information_pre_18, + connection_information_post_18, 1}, + {connection_information, connection_information_pre_18, + connection_information_post_18, 2}]). + -export([connection_information/1, - connection_information/2]). + connection_information_pre_18/1, + connection_information_post_18/1, + connection_information/2, + connection_information_pre_18/2, + connection_information_post_18/2]). connection_information(SslSocket) -> - try - ssl:connection_information(SslSocket) - catch - error:undef -> + code_version:update(?MODULE), + ssl_compat:connection_information(SslSocket). + +connection_information_post_18(SslSocket) -> + ssl:connection_information(SslSocket). + +connection_information_pre_18(SslSocket) -> + case ssl:connection_info(SslSocket) of + {ok, {ProtocolVersion, CipherSuite}} -> + {ok, [{protocol, ProtocolVersion}, + {cipher_suite, CipherSuite}]}; + {error, Reason} -> + {error, Reason} + end. + +connection_information(SslSocket, Items) -> + code_version:update(?MODULE), + ssl_compat:connection_information(SslSocket, Items). + +connection_information_post_18(SslSocket, Items) -> + ssl:connection_information(SslSocket, Items). + +connection_information_pre_18(SslSocket, Items) -> + WantProtocolVersion = lists:member(protocol, Items), + WantCipherSuite = lists:member(cipher_suite, Items), + if + WantProtocolVersion orelse WantCipherSuite -> case ssl:connection_info(SslSocket) of {ok, {ProtocolVersion, CipherSuite}} -> - {ok, [{protocol, ProtocolVersion}, - {cipher_suite, CipherSuite}]}; + filter_information_items(ProtocolVersion, + CipherSuite, + Items, + []); {error, Reason} -> {error, Reason} - end - end. - -connection_information(SslSocket, Items) -> - try - ssl:connection_information(SslSocket, Items) - catch - error:undef -> - WantProtocolVersion = lists:member(protocol, Items), - WantCipherSuite = lists:member(cipher_suite, Items), - if - WantProtocolVersion orelse WantCipherSuite -> - case ssl:connection_info(SslSocket) of - {ok, {ProtocolVersion, CipherSuite}} -> - filter_information_items(ProtocolVersion, - CipherSuite, - Items, - []); - {error, Reason} -> - {error, Reason} - end; - true -> - {ok, []} - end + end; + true -> + {ok, []} end. filter_information_items(ProtocolVersion, CipherSuite, [protocol | Rest], From 921bc1e8d76d17ec290cde080a69426f789f0b52 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Fri, 8 Jan 2016 17:13:30 +0100 Subject: [PATCH 22/45] Error handling fix * erl_syntax_lib:analyze_function throws syntax_error References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/code_version.erl b/src/code_version.erl index 9d2ed8f5..c5f997c5 100644 --- a/src/code_version.erl +++ b/src/code_version.erl @@ -186,11 +186,8 @@ replace_forms(Module, IsPost18, AbsCode) -> replace_function_forms(AbsCode, Fun) -> ReplaceFunction = fun(Tree) -> - case erl_syntax_lib:analyze_function(Tree) of - {_N, _A} = Function -> - Fun(Tree, Function); - _Other -> Tree - end + Function = erl_syntax_lib:analyze_function(Tree), + Fun(Tree, Function) end, Filter = fun(Tree) -> case erl_syntax:type(Tree) of From 3efb17910881a90dd56c96d50dc27e7de7ede5ce Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Fri, 22 Jan 2016 11:51:13 +0000 Subject: [PATCH 23/45] Use arity to rename, fix arity spec References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 8 ++++---- src/time_compat.erl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/code_version.erl b/src/code_version.erl index c5f997c5..54ad1c40 100644 --- a/src/code_version.erl +++ b/src/code_version.erl @@ -132,9 +132,9 @@ get_rename_pairs(false, VersionSupport) -> %% Pairs of {Renamed, OriginalName} functions get_name_pairs(true, VersionSupport) -> - [{Post, Orig} || {Orig, _Pre, Post, _Arity} <- VersionSupport]; + [{{Post, Arity}, Orig} || {Orig, _Pre, Post, Arity} <- VersionSupport]; get_name_pairs(false, VersionSupport) -> - [{Pre, Orig} || {Orig, Pre, _Post, _Arity} <- VersionSupport]. + [{{Pre, Arity}, Orig} || {Orig, Pre, _Post, Arity} <- VersionSupport]. delete_abstract_functions(ToDelete) -> fun(Tree, Function) -> @@ -147,10 +147,10 @@ delete_abstract_functions(ToDelete) -> end. rename_abstract_functions(ToRename, ToName) -> - fun(Tree, {Name, _Arity} = Function) -> + fun(Tree, Function) -> case lists:member(Function, ToRename) of true -> - FunctionName = proplists:get_value(Name, ToName), + FunctionName = proplists:get_value(Function, ToName), erl_syntax:function( erl_syntax:atom(FunctionName), erl_syntax:function_clauses(Tree)); diff --git a/src/time_compat.erl b/src/time_compat.erl index 2d91ccb9..0d789717 100644 --- a/src/time_compat.erl +++ b/src/time_compat.erl @@ -54,7 +54,7 @@ {os_system_time, os_system_time_pre_18, os_system_time_post_18, 1}, {time_offset, time_offset_pre_18, time_offset_post_18, 0}, {time_offset, time_offset_pre_18, time_offset_post_18, 1}, - {convert_time_unit, convert_time_unit_pre_18, convert_time_unit_post_18, 0}, + {convert_time_unit, convert_time_unit_pre_18, convert_time_unit_post_18, 3}, {timestamp, timestamp_pre_18, timestamp_post_18, 0}, {unique_integer, unique_integer_pre_18, unique_integer_post_18, 0}, {unique_integer, unique_integer_pre_18, unique_integer_post_18, 1}]). From 870c3c1780df137cf6d10143902c421a5287d0fb Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Fri, 22 Jan 2016 12:32:22 +0000 Subject: [PATCH 24/45] Support multiple Erlang versions References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 27 ++++++++++++++++++++------- src/ssl_compat.erl | 11 ++++++----- src/time_compat.erl | 28 +++++++++++++++------------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/code_version.erl b/src/code_version.erl index 54ad1c40..540175d3 100644 --- a/src/code_version.erl +++ b/src/code_version.erl @@ -32,7 +32,7 @@ %% The purpose of this functionality is to support the new time API introduced %% in ERTS 7.0, while providing compatibility with previous versions. %% -%% `Module` must contain an attribute `version_support` containing a list of +%% `Module` must contain an attribute `erlang_version_support` containing a list of %% tuples: {OriginalFuntion, PreErlang18Function, PostErlang18Function, Arity} %% %% All these new functions may be exported, and implemented as follows: @@ -56,7 +56,7 @@ -spec update(atom()) -> ok | no_return(). update(Module) -> AbsCode = get_abs_code(Module), - Forms = replace_forms(Module, get_otp_version() >= 18, AbsCode), + Forms = replace_forms(Module, get_otp_version(), AbsCode), Code = compile_forms(Forms), load_code(Module, Code). @@ -159,24 +159,37 @@ rename_abstract_functions(ToRename, ToName) -> end end. -replace_forms(Module, IsPost18, AbsCode) -> +replace_forms(Module, ErlangVersion, AbsCode) -> %% Obtain attribute containing the list of functions that must be updated Attr = Module:module_info(attributes), - VersionSupport = proplists:get_value(version_support, Attr), + VersionSupport = proplists:get_value(erlang_version_support, Attr), + {Pre, Post} = lists:splitwith(fun({Version, _Pairs}) -> + Version > ErlangVersion + end, VersionSupport), + %% Replace functions in two passes: replace for Erlang versions > current + %% first, Erlang versions =< current afterwards. + replace_version_forms( + true, replace_version_forms(false, AbsCode, get_version_functions(Pre)), + get_version_functions(Post)). + +get_version_functions(List) -> + lists:append([Pairs || {_Version, Pairs} <- List]). + +replace_version_forms(IsPost, AbsCode, VersionSupport) -> %% Get pairs of {Function, Arity} for the triggering functions, which %% are also the final function names. Original = get_original_pairs(VersionSupport), %% Get pairs of {Function, Arity} for the unused version - ToDelete = get_delete_pairs(IsPost18, VersionSupport), + ToDelete = get_delete_pairs(IsPost, VersionSupport), %% Delete original functions (those that trigger the code update) and %% the unused version ones DeleteFun = delete_abstract_functions(ToDelete ++ Original), AbsCode0 = replace_function_forms(AbsCode, DeleteFun), %% Get pairs of {Function, Arity} for the current version which must be %% renamed - ToRename = get_rename_pairs(IsPost18, VersionSupport), + ToRename = get_rename_pairs(IsPost, VersionSupport), %% Get paris of {Renamed, OriginalName} functions - ToName = get_name_pairs(IsPost18, VersionSupport), + ToName = get_name_pairs(IsPost, VersionSupport), %% Rename versioned functions with their final name RenameFun = rename_abstract_functions(ToRename, ToName), %% Remove exports of all versioned functions diff --git a/src/ssl_compat.erl b/src/ssl_compat.erl index 2c787011..a3085674 100644 --- a/src/ssl_compat.erl +++ b/src/ssl_compat.erl @@ -22,11 +22,12 @@ %% Declare versioned functions to allow dynamic code loading, %% depending on the Erlang version running. See 'code_version.erl' for details --version_support( - [{connection_information, connection_information_pre_18, - connection_information_post_18, 1}, - {connection_information, connection_information_pre_18, - connection_information_post_18, 2}]). +-erlang_version_support( + [{18, [{connection_information, connection_information_pre_18, + connection_information_post_18, 1}, + {connection_information, connection_information_pre_18, + connection_information_post_18, 2}]} + ]). -export([connection_information/1, connection_information_pre_18/1, diff --git a/src/time_compat.erl b/src/time_compat.erl index 0d789717..d6f71bd9 100644 --- a/src/time_compat.erl +++ b/src/time_compat.erl @@ -45,19 +45,21 @@ %% Declare versioned functions to allow dynamic code loading, %% depending on the Erlang version running. See 'code_version.erl' for details --version_support( - [{monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 0}, - {monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 1}, - {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 0}, - {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 1}, - {os_system_time, os_system_time_pre_18, os_system_time_post_18, 0}, - {os_system_time, os_system_time_pre_18, os_system_time_post_18, 1}, - {time_offset, time_offset_pre_18, time_offset_post_18, 0}, - {time_offset, time_offset_pre_18, time_offset_post_18, 1}, - {convert_time_unit, convert_time_unit_pre_18, convert_time_unit_post_18, 3}, - {timestamp, timestamp_pre_18, timestamp_post_18, 0}, - {unique_integer, unique_integer_pre_18, unique_integer_post_18, 0}, - {unique_integer, unique_integer_pre_18, unique_integer_post_18, 1}]). +-erlang_version_support( + [{18, + [{monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 0}, + {monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 1}, + {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 0}, + {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 1}, + {os_system_time, os_system_time_pre_18, os_system_time_post_18, 0}, + {os_system_time, os_system_time_pre_18, os_system_time_post_18, 1}, + {time_offset, time_offset_pre_18, time_offset_post_18, 0}, + {time_offset, time_offset_pre_18, time_offset_post_18, 1}, + {convert_time_unit, convert_time_unit_pre_18, convert_time_unit_post_18, 3}, + {timestamp, timestamp_pre_18, timestamp_post_18, 0}, + {unique_integer, unique_integer_pre_18, unique_integer_post_18, 0}, + {unique_integer, unique_integer_pre_18, unique_integer_post_18, 1}]} + ]). -export([monotonic_time/0, monotonic_time_pre_18/0, From 0cc1687bc02427c35b7ff42cd468247cbfd22b38 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Tue, 9 Feb 2016 08:05:05 +0000 Subject: [PATCH 25/45] Update doc References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/code_version.erl b/src/code_version.erl index 540175d3..32c70062 100644 --- a/src/code_version.erl +++ b/src/code_version.erl @@ -33,7 +33,10 @@ %% in ERTS 7.0, while providing compatibility with previous versions. %% %% `Module` must contain an attribute `erlang_version_support` containing a list of -%% tuples: {OriginalFuntion, PreErlang18Function, PostErlang18Function, Arity} +%% tuples: +%% +%% {ErlangVersion, [{OriginalFuntion, PreErlangVersionFunction, +%% PostErlangVersionFunction, Arity}]} %% %% All these new functions may be exported, and implemented as follows: %% @@ -41,11 +44,11 @@ %% code_version:update(?MODULE), %% ?MODULE:OriginalFunction(). %% -%% PostErlang18Function() -> +%% PostErlangVersionFunction() -> %% %% implementation using new time API %% .. %% -%% PreErlang18Function() -> +%% PreErlangVersionFunction() -> %% %% implementation using fallback solution %% .. %% From 503b501f5c8566da2d9ce6ab3508e34bba761e5d Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Tue, 9 Feb 2016 08:14:59 +0000 Subject: [PATCH 26/45] Avoid unnecesary try/catch References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/time_compat.erl | 61 ++++++++------------------------------------- 1 file changed, 10 insertions(+), 51 deletions(-) diff --git a/src/time_compat.erl b/src/time_compat.erl index d6f71bd9..2eabd1d9 100644 --- a/src/time_compat.erl +++ b/src/time_compat.erl @@ -116,21 +116,12 @@ monotonic_time(Unit) -> time_compat:monotonic_time(Unit). monotonic_time_post_18(Unit) -> - try - erlang:monotonic_time(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]) - end. + erlang:monotonic_time(Unit). monotonic_time_pre_18(Unit) -> %% Use Erlang system time as monotonic time STime = erlang_system_time_fallback(), - try - convert_time_unit_fallback(STime, native, Unit) - catch - error:bad_time_unit -> erlang:error(badarg, [Unit]) - end. + convert_time_unit_fallback(STime, native, Unit). erlang_system_time() -> code_version:update(?MODULE), @@ -147,20 +138,11 @@ erlang_system_time(Unit) -> time_compat:erlang_system_time(Unit). erlang_system_time_post_18(Unit) -> - try - erlang:system_time(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]) - end. + erlang:system_time(Unit). erlang_system_time_pre_18(Unit) -> STime = erlang_system_time_fallback(), - try - convert_time_unit_fallback(STime, native, Unit) - catch - error:bad_time_unit -> erlang:error(badarg, [Unit]) - end. + convert_time_unit_fallback(STime, native, Unit). os_system_time() -> code_version:update(?MODULE), @@ -177,20 +159,11 @@ os_system_time(Unit) -> time_compat:os_system_time(Unit). os_system_time_post_18(Unit) -> - try - os:system_time(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]) - end. + os:system_time(Unit). os_system_time_pre_18(Unit) -> STime = os_system_time_fallback(), - try - convert_time_unit_fallback(STime, native, Unit) - catch - error:bad_time_unit -> erlang:error(badarg, [Unit]) - end. + convert_time_unit_fallback(STime, native, Unit). time_offset() -> code_version:update(?MODULE), @@ -209,19 +182,10 @@ time_offset(Unit) -> time_compat:time_offset(Unit). time_offset_post_18(Unit) -> - try - erlang:time_offset(Unit) - catch - error:badarg -> - erlang:error(badarg, [Unit]) - end. + erlang:time_offset(Unit). time_offset_pre_18(Unit) -> - try - _ = integer_time_unit(Unit) - catch - error:bad_time_unit -> erlang:error(badarg, [Unit]) - end, + _ = integer_time_unit(Unit), %% Erlang system time and Erlang monotonic %% time are always aligned 0. @@ -272,12 +236,7 @@ unique_integer(Modifiers) -> time_compat:unique_integer(Modifiers). unique_integer_post_18(Modifiers) -> - try - erlang:unique_integer(Modifiers) - catch - error:badarg -> - erlang:error(badarg, [Modifiers]) - end. + erlang:unique_integer(Modifiers). unique_integer_pre_18(Modifiers) -> case is_valid_modifier_list(Modifiers) of @@ -360,7 +319,7 @@ integer_time_unit(micro_seconds) -> 1000*1000; integer_time_unit(milli_seconds) -> 1000; integer_time_unit(seconds) -> 1; integer_time_unit(I) when is_integer(I), I > 0 -> I; -integer_time_unit(BadRes) -> erlang:error(bad_time_unit, [BadRes]). +integer_time_unit(BadRes) -> erlang:error(badarg, [BadRes]). erlang_system_time_fallback() -> {MS, S, US} = erlang:now(), From 76de310034b8d342917e2ea18d2523c0f7e8cbb8 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Wed, 24 Feb 2016 16:54:09 +0000 Subject: [PATCH 27/45] Pass the arity right after the real function name References rabbitmq/rabbitmq-server#346. References rabbitmq/rabbitmq-server#347. --- src/code_version.erl | 18 +++++++++--------- src/ssl_compat.erl | 8 ++++---- src/time_compat.erl | 24 ++++++++++++------------ 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/code_version.erl b/src/code_version.erl index 32c70062..a7558bbb 100644 --- a/src/code_version.erl +++ b/src/code_version.erl @@ -35,8 +35,8 @@ %% `Module` must contain an attribute `erlang_version_support` containing a list of %% tuples: %% -%% {ErlangVersion, [{OriginalFuntion, PreErlangVersionFunction, -%% PostErlangVersionFunction, Arity}]} +%% {ErlangVersion, [{OriginalFuntion, Arity, PreErlangVersionFunction, +%% PostErlangVersionFunction}]} %% %% All these new functions may be exported, and implemented as follows: %% @@ -121,23 +121,23 @@ get_otp_version() -> end. get_original_pairs(VersionSupport) -> - [{Orig, Arity} || {Orig, _Pre, _Post, Arity} <- VersionSupport]. + [{Orig, Arity} || {Orig, Arity, _Pre, _Post} <- VersionSupport]. get_delete_pairs(true, VersionSupport) -> - [{Pre, Arity} || {_Orig, Pre, _Post, Arity} <- VersionSupport]; + [{Pre, Arity} || {_Orig, Arity, Pre, _Post} <- VersionSupport]; get_delete_pairs(false, VersionSupport) -> - [{Post, Arity} || {_Orig, _Pre, Post, Arity} <- VersionSupport]. + [{Post, Arity} || {_Orig, Arity, _Pre, Post} <- VersionSupport]. get_rename_pairs(true, VersionSupport) -> - [{Post, Arity} || {_Orig, _Pre, Post, Arity} <- VersionSupport]; + [{Post, Arity} || {_Orig, Arity, _Pre, Post} <- VersionSupport]; get_rename_pairs(false, VersionSupport) -> - [{Pre, Arity} || {_Orig, Pre, _Post, Arity} <- VersionSupport]. + [{Pre, Arity} || {_Orig, Arity, Pre, _Post} <- VersionSupport]. %% Pairs of {Renamed, OriginalName} functions get_name_pairs(true, VersionSupport) -> - [{{Post, Arity}, Orig} || {Orig, _Pre, Post, Arity} <- VersionSupport]; + [{{Post, Arity}, Orig} || {Orig, Arity, _Pre, Post} <- VersionSupport]; get_name_pairs(false, VersionSupport) -> - [{{Pre, Arity}, Orig} || {Orig, Pre, _Post, Arity} <- VersionSupport]. + [{{Pre, Arity}, Orig} || {Orig, Arity, Pre, _Post} <- VersionSupport]. delete_abstract_functions(ToDelete) -> fun(Tree, Function) -> diff --git a/src/ssl_compat.erl b/src/ssl_compat.erl index a3085674..e007667e 100644 --- a/src/ssl_compat.erl +++ b/src/ssl_compat.erl @@ -23,10 +23,10 @@ %% Declare versioned functions to allow dynamic code loading, %% depending on the Erlang version running. See 'code_version.erl' for details -erlang_version_support( - [{18, [{connection_information, connection_information_pre_18, - connection_information_post_18, 1}, - {connection_information, connection_information_pre_18, - connection_information_post_18, 2}]} + [{18, [{connection_information, 1, connection_information_pre_18, + connection_information_post_18}, + {connection_information, 2, connection_information_pre_18, + connection_information_post_18}]} ]). -export([connection_information/1, diff --git a/src/time_compat.erl b/src/time_compat.erl index 2eabd1d9..66044312 100644 --- a/src/time_compat.erl +++ b/src/time_compat.erl @@ -47,18 +47,18 @@ %% depending on the Erlang version running. See 'code_version.erl' for details -erlang_version_support( [{18, - [{monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 0}, - {monotonic_time, monotonic_time_pre_18, monotonic_time_post_18, 1}, - {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 0}, - {erlang_system_time, erlang_system_time_pre_18, erlang_system_time_post_18, 1}, - {os_system_time, os_system_time_pre_18, os_system_time_post_18, 0}, - {os_system_time, os_system_time_pre_18, os_system_time_post_18, 1}, - {time_offset, time_offset_pre_18, time_offset_post_18, 0}, - {time_offset, time_offset_pre_18, time_offset_post_18, 1}, - {convert_time_unit, convert_time_unit_pre_18, convert_time_unit_post_18, 3}, - {timestamp, timestamp_pre_18, timestamp_post_18, 0}, - {unique_integer, unique_integer_pre_18, unique_integer_post_18, 0}, - {unique_integer, unique_integer_pre_18, unique_integer_post_18, 1}]} + [{monotonic_time, 0, monotonic_time_pre_18, monotonic_time_post_18}, + {monotonic_time, 1, monotonic_time_pre_18, monotonic_time_post_18}, + {erlang_system_time, 0, erlang_system_time_pre_18, erlang_system_time_post_18}, + {erlang_system_time, 1, erlang_system_time_pre_18, erlang_system_time_post_18}, + {os_system_time, 0, os_system_time_pre_18, os_system_time_post_18}, + {os_system_time, 1, os_system_time_pre_18, os_system_time_post_18}, + {time_offset, 0, time_offset_pre_18, time_offset_post_18}, + {time_offset, 1, time_offset_pre_18, time_offset_post_18}, + {convert_time_unit, 3, convert_time_unit_pre_18, convert_time_unit_post_18}, + {timestamp, 0, timestamp_pre_18, timestamp_post_18}, + {unique_integer, 0, unique_integer_pre_18, unique_integer_post_18}, + {unique_integer, 1, unique_integer_pre_18, unique_integer_post_18}]} ]). -export([monotonic_time/0, From 85d57e1880217ed12d1234ceaddec39b75fd9bd5 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Thu, 25 Feb 2016 15:26:19 +0000 Subject: [PATCH 28/45] Move rpc_call from rabbit/rabbit_cli to rabbit_misc --- include/rabbit_misc.hrl | 17 +++++++++++++++++ src/rabbit_misc.erl | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 include/rabbit_misc.hrl diff --git a/include/rabbit_misc.hrl b/include/rabbit_misc.hrl new file mode 100644 index 00000000..26dce4ba --- /dev/null +++ b/include/rabbit_misc.hrl @@ -0,0 +1,17 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-define(RPC_TIMEOUT, infinity). diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl index 52675762..4bce9e05 100644 --- a/src/rabbit_misc.erl +++ b/src/rabbit_misc.erl @@ -17,6 +17,7 @@ -module(rabbit_misc). -include("rabbit.hrl"). -include("rabbit_framing.hrl"). +-include("rabbit_misc.hrl"). -export([method_record_type/1, polite_pause/0, polite_pause/1]). -export([die/1, frame_error/2, amqp_error/4, quit/1, @@ -73,6 +74,7 @@ -export([get_env/3]). -export([get_channel_operation_timeout/0]). -export([random/1]). +-export([rpc_call/4, rpc_call/5, rpc_call/7]). %% Horrible macro to use in guards -define(IS_BENIGN_EXIT(R), @@ -263,6 +265,10 @@ -spec(get_env/3 :: (atom(), atom(), term()) -> term()). -spec(get_channel_operation_timeout/0 :: () -> non_neg_integer()). -spec(random/1 :: (non_neg_integer()) -> non_neg_integer()). +-spec(rpc_call/4 :: (node(), atom(), atom(), [any()]) -> any()). +-spec(rpc_call/5 :: (node(), atom(), atom(), [any()], number()) -> any()). +-spec(rpc_call/7 :: (node(), atom(), atom(), [any()], reference(), pid(), + number()) -> any()). -endif. @@ -1160,6 +1166,24 @@ random(N) -> end, random:uniform(N). +%% Moved from rabbit/src/rabbit_cli.erl +%% If the server we are talking to has non-standard net_ticktime, and +%% our connection lasts a while, we could get disconnected because of +%% a timeout unless we set our ticktime to be the same. So let's do +%% that. +rpc_call(Node, Mod, Fun, Args) -> + rpc_call(Node, Mod, Fun, Args, ?RPC_TIMEOUT). + +rpc_call(Node, Mod, Fun, Args, Timeout) -> + case rpc:call(Node, net_kernel, get_net_ticktime, [], Timeout) of + {badrpc, _} = E -> E; + Time -> net_kernel:set_net_ticktime(Time, 0), + rpc:call(Node, Mod, Fun, Args, Timeout) + end. + +rpc_call(Node, Mod, Fun, Args, Ref, Pid, Timeout) -> + rpc_call(Node, Mod, Fun, Args++[Ref, Pid], Timeout). + %% ------------------------------------------------------------------------- %% Begin copypasta from gen_server2.erl From b836528d82a19a99f7c3e464838dd84f9bc9516f Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Thu, 25 Feb 2016 15:34:21 +0000 Subject: [PATCH 29/45] Add node health checks --- src/rabbit_health_check.erl | 94 +++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/rabbit_health_check.erl diff --git a/src/rabbit_health_check.erl b/src/rabbit_health_check.erl new file mode 100644 index 00000000..b87c8aac --- /dev/null +++ b/src/rabbit_health_check.erl @@ -0,0 +1,94 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% +-module(rabbit_health_check). + +-export([node/1]). + +-define(NODE_HEALTH_CHECK_TIMEOUT, 70000). + +-ifdef(use_specs). +-spec(node/1 :: (node()) -> 'true' | no_return()). +-endif. + +%%---------------------------------------------------------------------------- +%% External functions +%%---------------------------------------------------------------------------- +node(Node) -> + node_health_check(Node, is_running), + node_health_check(Node, list_channels), + node_health_check(Node, list_queues), + node_health_check(Node, alarms). + +%%---------------------------------------------------------------------------- +%% Internal functions +%%---------------------------------------------------------------------------- +node_health_check(Node, is_running) -> + node_health_check( + Node, {rabbit, is_running, []}, + fun(true) -> + true; + (false) -> + throw({node_is_ko, "rabbit application is not running"}) + end); +node_health_check(Node, list_channels) -> + node_health_check( + Node, {rabbit_channel, info_all, [[pid]]}, + fun(L) when is_list(L) -> + true; + (Other) -> + ErrorMsg = io_lib:format("list_channels unexpected output: ~p", + [Other]), + throw({node_is_ko, ErrorMsg}) + end); +node_health_check(Node, list_queues) -> + node_health_check( + Node, {rabbit_amqqueue, info_all, [[pid]]}, + fun(L) when is_list(L) -> + true; + (Other) -> + ErrorMsg = io_lib:format("list_queues unexpected output: ~p", + [Other]), + throw({node_is_ko, ErrorMsg}) + end); +node_health_check(Node, alarms) -> + node_health_check( + Node, {rabbit, status, []}, + fun(Props) -> + case proplists:get_value(alarms, Props) of + [] -> + true; + Alarms -> + ErrorMsg = io_lib:format("alarms raised ~p", [Alarms]), + throw({node_is_ko, ErrorMsg}) + end + end). + +node_health_check(Node, {M, F, A}, Fun) -> + case rabbit_misc:rpc_call(Node, M, F, A, ?NODE_HEALTH_CHECK_TIMEOUT) of + {badrpc, timeout} -> + ErrorMsg = io_lib:format( + "health check of node ~p fails: timed out (~p ms)", + [Node, ?NODE_HEALTH_CHECK_TIMEOUT]), + throw({node_is_ko, ErrorMsg}); + {badrpc, Reason} -> + ErrorMsg = io_lib:format( + "health check of node ~p fails: ~p", [Node, Reason]), + throw({node_is_ko, ErrorMsg}); + Other -> + Fun(Other) + end. + + From 92ae50e5964d4f079c7b2abed1caaa8ab54a439b Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Fri, 26 Feb 2016 17:20:46 +0000 Subject: [PATCH 30/45] Return Unix error codes in node health check --- src/rabbit_health_check.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/rabbit_health_check.erl b/src/rabbit_health_check.erl index b87c8aac..1202016c 100644 --- a/src/rabbit_health_check.erl +++ b/src/rabbit_health_check.erl @@ -41,7 +41,7 @@ node_health_check(Node, is_running) -> fun(true) -> true; (false) -> - throw({node_is_ko, "rabbit application is not running"}) + throw({node_is_ko, "rabbit application is not running", 70}) end); node_health_check(Node, list_channels) -> node_health_check( @@ -51,7 +51,7 @@ node_health_check(Node, list_channels) -> (Other) -> ErrorMsg = io_lib:format("list_channels unexpected output: ~p", [Other]), - throw({node_is_ko, ErrorMsg}) + throw({node_is_ko, ErrorMsg, 70}) end); node_health_check(Node, list_queues) -> node_health_check( @@ -61,7 +61,7 @@ node_health_check(Node, list_queues) -> (Other) -> ErrorMsg = io_lib:format("list_queues unexpected output: ~p", [Other]), - throw({node_is_ko, ErrorMsg}) + throw({node_is_ko, ErrorMsg, 70}) end); node_health_check(Node, alarms) -> node_health_check( @@ -72,7 +72,7 @@ node_health_check(Node, alarms) -> true; Alarms -> ErrorMsg = io_lib:format("alarms raised ~p", [Alarms]), - throw({node_is_ko, ErrorMsg}) + throw({node_is_ko, ErrorMsg, 70}) end end). @@ -82,11 +82,15 @@ node_health_check(Node, {M, F, A}, Fun) -> ErrorMsg = io_lib:format( "health check of node ~p fails: timed out (~p ms)", [Node, ?NODE_HEALTH_CHECK_TIMEOUT]), - throw({node_is_ko, ErrorMsg}); + throw({node_is_ko, ErrorMsg, 70}); + {badrpc, nodedown} -> + ErrorMsg = io_lib:format( + "health check of node ~p fails: nodedown", [Node]), + throw({node_is_ko, ErrorMsg, 68}); {badrpc, Reason} -> ErrorMsg = io_lib:format( "health check of node ~p fails: ~p", [Node, Reason]), - throw({node_is_ko, ErrorMsg}); + throw({node_is_ko, ErrorMsg, 70}); Other -> Fun(Other) end. From 2437a4a4d514e1e2c1fa9a21d08a3ca673d6c1ac Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Mon, 29 Feb 2016 02:03:50 +0300 Subject: [PATCH 31/45] Wording --- src/rabbit_health_check.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_health_check.erl b/src/rabbit_health_check.erl index 1202016c..acef9d94 100644 --- a/src/rabbit_health_check.erl +++ b/src/rabbit_health_check.erl @@ -71,7 +71,7 @@ node_health_check(Node, alarms) -> [] -> true; Alarms -> - ErrorMsg = io_lib:format("alarms raised ~p", [Alarms]), + ErrorMsg = io_lib:format("resource alarm(s) in effect:~p", [Alarms]), throw({node_is_ko, ErrorMsg, 70}) end end). From 0ff4927bfc296205a6338f1aee4775dfe1399730 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Wed, 2 Mar 2016 15:44:36 +0000 Subject: [PATCH 32/45] Ignores notify_queues/1 result and removes its matches to 'ok'. Ref: #63 --- src/rabbit_channel.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index 13520d93..f312acf8 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -608,11 +608,11 @@ handle_pre_hibernate(State) -> {hibernate, rabbit_event:stop_stats_timer(State, #ch.stats_timer)}. terminate(Reason, State) -> - {Res, _State1} = notify_queues(State), + {_Res, _State1} = notify_queues(State), case Reason of - normal -> ok = Res; - shutdown -> ok = Res; - {shutdown, _Term} -> ok = Res; + normal -> ok; + shutdown -> ok; + {shutdown, _Term} -> ok; _ -> ok end, pg_local:leave(rabbit_channels, self()), From 9248ec5d194f4a44620d6b0e1da22298a7183f83 Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Wed, 2 Mar 2016 15:55:19 +0000 Subject: [PATCH 33/45] Completely ignore unused channel's termination reason. --- src/rabbit_channel.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl index f312acf8..78505934 100644 --- a/src/rabbit_channel.erl +++ b/src/rabbit_channel.erl @@ -607,14 +607,8 @@ handle_pre_hibernate(State) -> end), {hibernate, rabbit_event:stop_stats_timer(State, #ch.stats_timer)}. -terminate(Reason, State) -> +terminate(_Reason, State) -> {_Res, _State1} = notify_queues(State), - case Reason of - normal -> ok; - shutdown -> ok; - {shutdown, _Term} -> ok; - _ -> ok - end, pg_local:leave(rabbit_channels, self()), rabbit_event:if_enabled(State, #ch.stats_timer, fun() -> emit_stats(State) end), From 0a966e67c96211b9d2ad02ceaa221319c5016c2a Mon Sep 17 00:00:00 2001 From: Brandon Shroyer Date: Wed, 2 Mar 2016 13:52:52 -0500 Subject: [PATCH 34/45] Move types and auth_backend modules over from rabbitmq-server. * Remove modules from rabbitmq-server and add them to rabbitmq-common. --- src/rabbit_auth_backend_dummy.erl | 48 ++++ src/rabbit_auth_backend_internal.erl | 400 +++++++++++++++++++++++++++ src/rabbit_types.erl | 168 +++++++++++ 3 files changed, 616 insertions(+) create mode 100644 src/rabbit_auth_backend_dummy.erl create mode 100644 src/rabbit_auth_backend_internal.erl create mode 100644 src/rabbit_types.erl diff --git a/src/rabbit_auth_backend_dummy.erl b/src/rabbit_auth_backend_dummy.erl new file mode 100644 index 00000000..0077b4c9 --- /dev/null +++ b/src/rabbit_auth_backend_dummy.erl @@ -0,0 +1,48 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_auth_backend_dummy). +-include("rabbit.hrl"). + +-behaviour(rabbit_authn_backend). +-behaviour(rabbit_authz_backend). + +-export([user/0]). +-export([user_login_authentication/2, user_login_authorization/1, + check_vhost_access/3, check_resource_access/3]). + +-ifdef(use_specs). + +-spec(user/0 :: () -> rabbit_types:user()). + +-endif. + +%% A user to be used by the direct client when permission checks are +%% not needed. This user can do anything AMQPish. +user() -> #user{username = <<"none">>, + tags = [], + authz_backends = [{?MODULE, none}]}. + +%% Implementation of rabbit_auth_backend + +user_login_authentication(_, _) -> + {refused, "cannot log in conventionally as dummy user", []}. + +user_login_authorization(_) -> + {refused, "cannot log in conventionally as dummy user", []}. + +check_vhost_access(#auth_user{}, _VHostPath, _Sock) -> true. +check_resource_access(#auth_user{}, #resource{}, _Permission) -> true. diff --git a/src/rabbit_auth_backend_internal.erl b/src/rabbit_auth_backend_internal.erl new file mode 100644 index 00000000..d7705d8e --- /dev/null +++ b/src/rabbit_auth_backend_internal.erl @@ -0,0 +1,400 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_auth_backend_internal). +-include("rabbit.hrl"). + +-behaviour(rabbit_authn_backend). +-behaviour(rabbit_authz_backend). + +-export([user_login_authentication/2, user_login_authorization/1, + check_vhost_access/3, check_resource_access/3]). + +-export([add_user/2, delete_user/1, lookup_user/1, + change_password/2, clear_password/1, + hash_password/2, change_password_hash/2, change_password_hash/3, + set_tags/2, set_permissions/5, clear_permissions/2]). +-export([user_info_keys/0, perms_info_keys/0, + user_perms_info_keys/0, vhost_perms_info_keys/0, + user_vhost_perms_info_keys/0, + list_users/0, list_users/2, list_permissions/0, + list_user_permissions/1, list_user_permissions/3, + list_vhost_permissions/1, list_vhost_permissions/3, + list_user_vhost_permissions/2]). + +%% for testing +-export([hashing_module_for_user/1]). + +%%---------------------------------------------------------------------------- + +-ifdef(use_specs). + +-type(regexp() :: binary()). + +-spec(add_user/2 :: (rabbit_types:username(), rabbit_types:password()) -> 'ok'). +-spec(delete_user/1 :: (rabbit_types:username()) -> 'ok'). +-spec(lookup_user/1 :: (rabbit_types:username()) + -> rabbit_types:ok(rabbit_types:internal_user()) + | rabbit_types:error('not_found')). +-spec(change_password/2 :: (rabbit_types:username(), rabbit_types:password()) + -> 'ok'). +-spec(clear_password/1 :: (rabbit_types:username()) -> 'ok'). +-spec(hash_password/2 :: (module(), rabbit_types:password()) + -> rabbit_types:password_hash()). +-spec(change_password_hash/2 :: (rabbit_types:username(), + rabbit_types:password_hash()) -> 'ok'). +-spec(set_tags/2 :: (rabbit_types:username(), [atom()]) -> 'ok'). +-spec(set_permissions/5 ::(rabbit_types:username(), rabbit_types:vhost(), + regexp(), regexp(), regexp()) -> 'ok'). +-spec(clear_permissions/2 :: (rabbit_types:username(), rabbit_types:vhost()) + -> 'ok'). +-spec(user_info_keys/0 :: () -> rabbit_types:info_keys()). +-spec(perms_info_keys/0 :: () -> rabbit_types:info_keys()). +-spec(user_perms_info_keys/0 :: () -> rabbit_types:info_keys()). +-spec(vhost_perms_info_keys/0 :: () -> rabbit_types:info_keys()). +-spec(user_vhost_perms_info_keys/0 :: () -> rabbit_types:info_keys()). +-spec(list_users/0 :: () -> [rabbit_types:infos()]). +-spec(list_users/2 :: (reference(), pid()) -> 'ok'). +-spec(list_permissions/0 :: () -> [rabbit_types:infos()]). +-spec(list_user_permissions/1 :: + (rabbit_types:username()) -> [rabbit_types:infos()]). +-spec(list_user_permissions/3 :: + (rabbit_types:username(), reference(), pid()) -> 'ok'). +-spec(list_vhost_permissions/1 :: + (rabbit_types:vhost()) -> [rabbit_types:infos()]). +-spec(list_vhost_permissions/3 :: + (rabbit_types:vhost(), reference(), pid()) -> 'ok'). +-spec(list_user_vhost_permissions/2 :: + (rabbit_types:username(), rabbit_types:vhost()) + -> [rabbit_types:infos()]). + +-endif. + +%%---------------------------------------------------------------------------- +%% Implementation of rabbit_auth_backend + +%% Returns a password hashing module for the user record provided. If +%% there is no information in the record, we consider it to be legacy +%% (inserted by a version older than 3.6.0) and fall back to MD5, the +%% now obsolete hashing function. +hashing_module_for_user(#internal_user{ + hashing_algorithm = ModOrUndefined}) -> + rabbit_password:hashing_mod(ModOrUndefined). + +user_login_authentication(Username, []) -> + internal_check_user_login(Username, fun(_) -> true end); +user_login_authentication(Username, [{password, Cleartext}]) -> + internal_check_user_login( + Username, + fun (#internal_user{password_hash = <>} = U) -> + Hash =:= rabbit_password:salted_hash( + hashing_module_for_user(U), Salt, Cleartext); + (#internal_user{}) -> + false + end); +user_login_authentication(Username, AuthProps) -> + exit({unknown_auth_props, Username, AuthProps}). + +user_login_authorization(Username) -> + case user_login_authentication(Username, []) of + {ok, #auth_user{impl = Impl, tags = Tags}} -> {ok, Impl, Tags}; + Else -> Else + end. + +internal_check_user_login(Username, Fun) -> + Refused = {refused, "user '~s' - invalid credentials", [Username]}, + case lookup_user(Username) of + {ok, User = #internal_user{tags = Tags}} -> + case Fun(User) of + true -> {ok, #auth_user{username = Username, + tags = Tags, + impl = none}}; + _ -> Refused + end; + {error, not_found} -> + Refused + end. + +check_vhost_access(#auth_user{username = Username}, VHostPath, _Sock) -> + case mnesia:dirty_read({rabbit_user_permission, + #user_vhost{username = Username, + virtual_host = VHostPath}}) of + [] -> false; + [_R] -> true + end. + +check_resource_access(#auth_user{username = Username}, + #resource{virtual_host = VHostPath, name = Name}, + Permission) -> + case mnesia:dirty_read({rabbit_user_permission, + #user_vhost{username = Username, + virtual_host = VHostPath}}) of + [] -> + false; + [#user_permission{permission = P}] -> + PermRegexp = case element(permission_index(Permission), P) of + %% <<"^$">> breaks Emacs' erlang mode + <<"">> -> <<$^, $$>>; + RE -> RE + end, + case re:run(Name, PermRegexp, [{capture, none}]) of + match -> true; + nomatch -> false + end + end. + +permission_index(configure) -> #permission.configure; +permission_index(write) -> #permission.write; +permission_index(read) -> #permission.read. + +%%---------------------------------------------------------------------------- +%% Manipulation of the user database + +add_user(Username, Password) -> + rabbit_log:info("Creating user '~s'~n", [Username]), + %% hash_password will pick the hashing function configured for us + %% but we also need to store a hint as part of the record, so we + %% retrieve it here one more time + HashingMod = rabbit_password:hashing_mod(), + User = #internal_user{username = Username, + password_hash = hash_password(HashingMod, Password), + tags = [], + hashing_algorithm = HashingMod}, + R = rabbit_misc:execute_mnesia_transaction( + fun () -> + case mnesia:wread({rabbit_user, Username}) of + [] -> + ok = mnesia:write(rabbit_user, User, write); + _ -> + mnesia:abort({user_already_exists, Username}) + end + end), + rabbit_event:notify(user_created, [{name, Username}]), + R. + +delete_user(Username) -> + rabbit_log:info("Deleting user '~s'~n", [Username]), + R = rabbit_misc:execute_mnesia_transaction( + rabbit_misc:with_user( + Username, + fun () -> + ok = mnesia:delete({rabbit_user, Username}), + [ok = mnesia:delete_object( + rabbit_user_permission, R, write) || + R <- mnesia:match_object( + rabbit_user_permission, + #user_permission{user_vhost = #user_vhost{ + username = Username, + virtual_host = '_'}, + permission = '_'}, + write)], + ok + end)), + rabbit_event:notify(user_deleted, [{name, Username}]), + R. + +lookup_user(Username) -> + rabbit_misc:dirty_read({rabbit_user, Username}). + +change_password(Username, Password) -> + rabbit_log:info("Changing password for '~s'~n", [Username]), + HashingAlgorithm = rabbit_password:hashing_mod(), + R = change_password_hash(Username, + hash_password(rabbit_password:hashing_mod(), + Password), + HashingAlgorithm), + rabbit_event:notify(user_password_changed, [{name, Username}]), + R. + +clear_password(Username) -> + rabbit_log:info("Clearing password for '~s'~n", [Username]), + R = change_password_hash(Username, <<"">>), + rabbit_event:notify(user_password_cleared, [{name, Username}]), + R. + +hash_password(HashingMod, Cleartext) -> + rabbit_password:hash(HashingMod, Cleartext). + +change_password_hash(Username, PasswordHash) -> + change_password_hash(Username, PasswordHash, rabbit_password:hashing_mod()). + + +change_password_hash(Username, PasswordHash, HashingAlgorithm) -> + update_user(Username, fun(User) -> + User#internal_user{ + password_hash = PasswordHash, + hashing_algorithm = HashingAlgorithm } + end). + +set_tags(Username, Tags) -> + rabbit_log:info("Setting user tags for user '~s' to ~p~n", + [Username, Tags]), + R = update_user(Username, fun(User) -> + User#internal_user{tags = Tags} + end), + rabbit_event:notify(user_tags_set, [{name, Username}, {tags, Tags}]), + R. + +set_permissions(Username, VHostPath, ConfigurePerm, WritePerm, ReadPerm) -> + rabbit_log:info("Setting permissions for " + "'~s' in '~s' to '~s', '~s', '~s'~n", + [Username, VHostPath, ConfigurePerm, WritePerm, ReadPerm]), + lists:map( + fun (RegexpBin) -> + Regexp = binary_to_list(RegexpBin), + case re:compile(Regexp) of + {ok, _} -> ok; + {error, Reason} -> throw({error, {invalid_regexp, + Regexp, Reason}}) + end + end, [ConfigurePerm, WritePerm, ReadPerm]), + R = rabbit_misc:execute_mnesia_transaction( + rabbit_misc:with_user_and_vhost( + Username, VHostPath, + fun () -> ok = mnesia:write( + rabbit_user_permission, + #user_permission{user_vhost = #user_vhost{ + username = Username, + virtual_host = VHostPath}, + permission = #permission{ + configure = ConfigurePerm, + write = WritePerm, + read = ReadPerm}}, + write) + end)), + rabbit_event:notify(permission_created, [{user, Username}, + {vhost, VHostPath}, + {configure, ConfigurePerm}, + {write, WritePerm}, + {read, ReadPerm}]), + R. + +clear_permissions(Username, VHostPath) -> + R = rabbit_misc:execute_mnesia_transaction( + rabbit_misc:with_user_and_vhost( + Username, VHostPath, + fun () -> + ok = mnesia:delete({rabbit_user_permission, + #user_vhost{username = Username, + virtual_host = VHostPath}}) + end)), + rabbit_event:notify(permission_deleted, [{user, Username}, + {vhost, VHostPath}]), + R. + + +update_user(Username, Fun) -> + rabbit_misc:execute_mnesia_transaction( + rabbit_misc:with_user( + Username, + fun () -> + {ok, User} = lookup_user(Username), + ok = mnesia:write(rabbit_user, Fun(User), write) + end)). + +%%---------------------------------------------------------------------------- +%% Listing + +-define(PERMS_INFO_KEYS, [configure, write, read]). +-define(USER_INFO_KEYS, [user, tags]). + +user_info_keys() -> ?USER_INFO_KEYS. + +perms_info_keys() -> [user, vhost | ?PERMS_INFO_KEYS]. +vhost_perms_info_keys() -> [user | ?PERMS_INFO_KEYS]. +user_perms_info_keys() -> [vhost | ?PERMS_INFO_KEYS]. +user_vhost_perms_info_keys() -> ?PERMS_INFO_KEYS. + +list_users() -> + [extract_internal_user_params(U) || + U <- mnesia:dirty_match_object(rabbit_user, #internal_user{_ = '_'})]. + +list_users(Ref, AggregatorPid) -> + rabbit_control_misc:emitting_map( + AggregatorPid, Ref, + fun(U) -> extract_internal_user_params(U) end, + mnesia:dirty_match_object(rabbit_user, #internal_user{_ = '_'})). + +list_permissions() -> + list_permissions(perms_info_keys(), match_user_vhost('_', '_')). + +list_permissions(Keys, QueryThunk) -> + [extract_user_permission_params(Keys, U) || + %% TODO: use dirty ops instead + U <- rabbit_misc:execute_mnesia_transaction(QueryThunk)]. + +list_permissions(Keys, QueryThunk, Ref, AggregatorPid) -> + rabbit_control_misc:emitting_map( + AggregatorPid, Ref, fun(U) -> extract_user_permission_params(Keys, U) end, + %% TODO: use dirty ops instead + rabbit_misc:execute_mnesia_transaction(QueryThunk)). + +filter_props(Keys, Props) -> [T || T = {K, _} <- Props, lists:member(K, Keys)]. + +list_user_permissions(Username) -> + list_permissions( + user_perms_info_keys(), + rabbit_misc:with_user(Username, match_user_vhost(Username, '_'))). + +list_user_permissions(Username, Ref, AggregatorPid) -> + list_permissions( + user_perms_info_keys(), + rabbit_misc:with_user(Username, match_user_vhost(Username, '_')), + Ref, AggregatorPid). + +list_vhost_permissions(VHostPath) -> + list_permissions( + vhost_perms_info_keys(), + rabbit_vhost:with(VHostPath, match_user_vhost('_', VHostPath))). + +list_vhost_permissions(VHostPath, Ref, AggregatorPid) -> + list_permissions( + vhost_perms_info_keys(), + rabbit_vhost:with(VHostPath, match_user_vhost('_', VHostPath)), + Ref, AggregatorPid). + +list_user_vhost_permissions(Username, VHostPath) -> + list_permissions( + user_vhost_perms_info_keys(), + rabbit_misc:with_user_and_vhost( + Username, VHostPath, match_user_vhost(Username, VHostPath))). + +extract_user_permission_params(Keys, #user_permission{ + user_vhost = + #user_vhost{username = Username, + virtual_host = VHostPath}, + permission = #permission{ + configure = ConfigurePerm, + write = WritePerm, + read = ReadPerm}}) -> + filter_props(Keys, [{user, Username}, + {vhost, VHostPath}, + {configure, ConfigurePerm}, + {write, WritePerm}, + {read, ReadPerm}]). + +extract_internal_user_params(#internal_user{username = Username, tags = Tags}) -> + [{user, Username}, {tags, Tags}]. + +match_user_vhost(Username, VHostPath) -> + fun () -> mnesia:match_object( + rabbit_user_permission, + #user_permission{user_vhost = #user_vhost{ + username = Username, + virtual_host = VHostPath}, + permission = '_'}, + read) + end. diff --git a/src/rabbit_types.erl b/src/rabbit_types.erl new file mode 100644 index 00000000..3dcb63cb --- /dev/null +++ b/src/rabbit_types.erl @@ -0,0 +1,168 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rabbit_types). + +-include("rabbit.hrl"). + +-ifdef(use_specs). + +-export_type([maybe/1, info/0, infos/0, info_key/0, info_keys/0, + message/0, msg_id/0, basic_message/0, + delivery/0, content/0, decoded_content/0, undecoded_content/0, + unencoded_content/0, encoded_content/0, message_properties/0, + vhost/0, ctag/0, amqp_error/0, r/1, r2/2, r3/3, listener/0, + binding/0, binding_source/0, binding_destination/0, + amqqueue/0, exchange/0, + connection/0, protocol/0, auth_user/0, user/0, internal_user/0, + username/0, password/0, password_hash/0, + ok/1, error/1, ok_or_error/1, ok_or_error2/2, ok_pid_or_error/0, + channel_exit/0, connection_exit/0, mfargs/0, proc_name/0, + proc_type_and_name/0, timestamp/0]). + +-type(maybe(T) :: T | 'none'). +-type(timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}). +-type(vhost() :: binary()). +-type(ctag() :: binary()). + +%% TODO: make this more precise by tying specific class_ids to +%% specific properties +-type(undecoded_content() :: + #content{class_id :: rabbit_framing:amqp_class_id(), + properties :: 'none', + properties_bin :: binary(), + payload_fragments_rev :: [binary()]} | + #content{class_id :: rabbit_framing:amqp_class_id(), + properties :: rabbit_framing:amqp_property_record(), + properties_bin :: 'none', + payload_fragments_rev :: [binary()]}). +-type(unencoded_content() :: undecoded_content()). +-type(decoded_content() :: + #content{class_id :: rabbit_framing:amqp_class_id(), + properties :: rabbit_framing:amqp_property_record(), + properties_bin :: maybe(binary()), + payload_fragments_rev :: [binary()]}). +-type(encoded_content() :: + #content{class_id :: rabbit_framing:amqp_class_id(), + properties :: maybe(rabbit_framing:amqp_property_record()), + properties_bin :: binary(), + payload_fragments_rev :: [binary()]}). +-type(content() :: undecoded_content() | decoded_content()). +-type(msg_id() :: rabbit_guid:guid()). +-type(basic_message() :: + #basic_message{exchange_name :: rabbit_exchange:name(), + routing_keys :: [rabbit_router:routing_key()], + content :: content(), + id :: msg_id(), + is_persistent :: boolean()}). +-type(message() :: basic_message()). +-type(delivery() :: + #delivery{mandatory :: boolean(), + sender :: pid(), + message :: message()}). +-type(message_properties() :: + #message_properties{expiry :: pos_integer() | 'undefined', + needs_confirming :: boolean()}). + +-type(info_key() :: atom()). +-type(info_keys() :: [info_key()]). + +-type(info() :: {info_key(), any()}). +-type(infos() :: [info()]). + +-type(amqp_error() :: + #amqp_error{name :: rabbit_framing:amqp_exception(), + explanation :: string(), + method :: rabbit_framing:amqp_method_name()}). + +-type(r(Kind) :: + r2(vhost(), Kind)). +-type(r2(VirtualHost, Kind) :: + r3(VirtualHost, Kind, rabbit_misc:resource_name())). +-type(r3(VirtualHost, Kind, Name) :: + #resource{virtual_host :: VirtualHost, + kind :: Kind, + name :: Name}). + +-type(listener() :: + #listener{node :: node(), + protocol :: atom(), + host :: rabbit_networking:hostname(), + port :: rabbit_networking:ip_port()}). + +-type(binding_source() :: rabbit_exchange:name()). +-type(binding_destination() :: rabbit_amqqueue:name() | rabbit_exchange:name()). + +-type(binding() :: + #binding{source :: rabbit_exchange:name(), + destination :: binding_destination(), + key :: rabbit_binding:key(), + args :: rabbit_framing:amqp_table()}). + +-type(amqqueue() :: + #amqqueue{name :: rabbit_amqqueue:name(), + durable :: boolean(), + auto_delete :: boolean(), + exclusive_owner :: rabbit_types:maybe(pid()), + arguments :: rabbit_framing:amqp_table(), + pid :: rabbit_types:maybe(pid()), + slave_pids :: [pid()]}). + +-type(exchange() :: + #exchange{name :: rabbit_exchange:name(), + type :: rabbit_exchange:type(), + durable :: boolean(), + auto_delete :: boolean(), + arguments :: rabbit_framing:amqp_table()}). + +-type(connection() :: pid()). + +-type(protocol() :: rabbit_framing:protocol()). + +-type(auth_user() :: + #auth_user{username :: username(), + tags :: [atom()], + impl :: any()}). + +-type(user() :: + #user{username :: username(), + tags :: [atom()], + authz_backends :: [{atom(), any()}]}). + +-type(internal_user() :: + #internal_user{username :: username(), + password_hash :: password_hash(), + tags :: [atom()]}). + +-type(username() :: binary()). +-type(password() :: binary()). +-type(password_hash() :: binary()). + +-type(ok(A) :: {'ok', A}). +-type(error(A) :: {'error', A}). +-type(ok_or_error(A) :: 'ok' | error(A)). +-type(ok_or_error2(A, B) :: ok(A) | error(B)). +-type(ok_pid_or_error() :: ok_or_error2(pid(), any())). + +-type(channel_exit() :: no_return()). +-type(connection_exit() :: no_return()). + +-type(mfargs() :: {atom(), atom(), [any()]}). + +-type(proc_name() :: term()). +-type(proc_type_and_name() :: {atom(), proc_name()}). + +-endif. % use_specs From 4cf38fc28a10d7d4602b7e8a29556cc937ffe75f Mon Sep 17 00:00:00 2001 From: Ayanda Dube Date: Mon, 7 Mar 2016 09:37:14 +0000 Subject: [PATCH 35/45] Updates queue_master_location/1 parameter spec from pid() to #amqqueue{}. Ref: #68 --- src/rabbit_queue_master_locator.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rabbit_queue_master_locator.erl b/src/rabbit_queue_master_locator.erl index a73a307c..0dc60183 100644 --- a/src/rabbit_queue_master_locator.erl +++ b/src/rabbit_queue_master_locator.erl @@ -19,7 +19,8 @@ -ifdef(use_specs). -callback description() -> [proplists:property()]. --callback queue_master_location(pid()) -> {'ok', node()} | {'error', term()}. +-callback queue_master_location(rabbit_types:amqqueue()) -> + {'ok', node()} | {'error', term()}. -else. From 68c9f7d20db19c5828d7469d9499be78dcecc913 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 9 Mar 2016 02:44:14 +0300 Subject: [PATCH 36/45] Emit stats unconditionally ...of connection (flow control) state. This makes it much easier to reason about flow control state when looking at the management UI or monitoring tools that poll HTTP API. Now that rabbitmq/rabbitmq-management#41 is merged, there are few arguments against always emitting stats. Fixes #679. --- src/rabbit_reader.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 73513f9a..85a258af 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -1465,13 +1465,7 @@ emit_stats(State) -> Infos = infos(?STATISTICS_KEYS, State), rabbit_event:notify(connection_stats, Infos), State1 = rabbit_event:reset_stats_timer(State, #v1.stats_timer), - %% If we emit an event which looks like we are in flow control, it's not a - %% good idea for it to be our last even if we go idle. Keep emitting - %% events, either we stay busy or we drop out of flow control. - case proplists:get_value(state, Infos) of - flow -> ensure_stats_timer(State1); - _ -> State1 - end. + ensure_stats_timer(State1). %% 1.0 stub -ifdef(use_specs). From 5b92eefcc9b6c46eb2f7f089e8966dfe30090e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Tue, 15 Mar 2016 23:44:21 +0100 Subject: [PATCH 37/45] rabbitmq-run.mk: Export $(MAKE) to child processes This is useful for the testsuites. --- mk/rabbitmq-run.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mk/rabbitmq-run.mk b/mk/rabbitmq-run.mk index 72ce8b04..9f11cd36 100644 --- a/mk/rabbitmq-run.mk +++ b/mk/rabbitmq-run.mk @@ -39,6 +39,9 @@ endif export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER +# We export MAKE to be sure scripts and tests use the proper command. +export MAKE + # We need to pass the location of codegen to the Java client ant # process. CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen From fbc8f3515ae050abbf91fe9bf30d0d10aaa81d20 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Wed, 16 Mar 2016 15:41:15 +0000 Subject: [PATCH 38/45] AMQP reader throttle refactoring Squashed commit of the following: commit 0843d71dadc6f717d94dac90985587433c849da1 Author: Daniil Fedotov Date: Wed Mar 16 15:05:26 2016 +0000 Set alarmes when connection is opened commit 67f34835023f5182c0836c1072e1026d177299f6 Author: Michael Klishin Date: Wed Mar 16 14:06:23 2016 +0000 Emit connection state as 'blocking' before we see a publish commit 0fc1d16b05ecad220c4c34b8867576cea967de47 Author: Michael Klishin Date: Wed Mar 16 11:24:22 2016 +0000 Reset throttle state when onblocking; make sure we not block consumers commit 327dfd2d2ac97b760102b49b61306b01e041f512 Author: Michael Klishin Date: Mon Mar 14 17:17:37 2016 +0000 Report connection state as "in flow" again if we are currently blocked by flow control or were blocked in the last 5s commit 505000136e699af3e5b813fbf850637c187786d1 Author: Michael Klishin Date: Mon Mar 14 16:52:55 2016 +0000 Continued refactoring, per discussion with @hairyhum commit 512bb3a3102de08319727a465cd996693978ead8 Author: Michael Klishin Date: Mon Mar 14 15:32:38 2016 +0000 Remove things that are not intended for this branch commit 6ee5fdff37a9b24ecdb01f7a28c29a2c5147c619 Author: Michael Klishin Date: Mon Mar 14 15:26:19 2016 +0000 can_block => should_block commit f10308449a0cdc580b0798c3923664fdd14045c8 Author: Michael Klishin Date: Mon Mar 14 15:20:10 2016 +0000 Docs commit 46b4d4256304ad5a4a91d7a27717723602411f4d Author: Michael Klishin Date: Mon Mar 14 15:15:06 2016 +0000 Docs commit 30111a6991716d51ed126f26deebf4a072ddb782 Merge: 457eb87 76ec3d9 Author: Michael Klishin Date: Fri Mar 11 23:58:29 2016 +0300 Merge branch 'master' into rabbitmq-common-reader-state commit 457eb875c5a837fb9fdc3dc669537183cb559897 Author: Daniil Fedotov Date: Mon Mar 7 17:46:46 2016 +0000 Beter names for blocking params commit 1d85ed27382fbb171528cc0607b05ddb410b1081 Author: Daniil Fedotov Date: Tue Mar 1 18:29:21 2016 +0000 Blocked message formatting commit 8eede1a8c20c8475007ab336689b810fd1f50302 Author: Daniil Fedotov Date: Tue Mar 1 18:00:06 2016 +0000 correct api commit 9bd961fec0327945e73eda72ba0c693eea88cbfb Author: Daniil Fedotov Date: Tue Mar 1 13:14:38 2016 +0000 Fix throttle state commit 650ad3224bbc96b26cbaea4c46f7beb41bdc4630 Author: Daniil Fedotov Date: Fri Feb 26 17:45:28 2016 +0000 Extended throttle. Removed blocking state commit dc928f1325699ebb1e1403435fb245bb8a3b8967 Author: Daniil Fedotov Date: Fri Feb 26 13:03:16 2016 +0000 rabbit_reader state refactoring --- src/rabbit_reader.erl | 248 ++++++++++++++++++++++++++++-------------- 1 file changed, 167 insertions(+), 81 deletions(-) diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl index 6d3b38cf..0cc0bc1e 100644 --- a/src/rabbit_reader.erl +++ b/src/rabbit_reader.erl @@ -144,12 +144,19 @@ connected_at}). -record(throttle, { - %% list of active alarms - alarmed_by, - %% flow | resource - last_blocked_by, %% never | timestamp() - last_blocked_at + last_blocked_at, + %% a set of the reasons why we are + %% blocked: {resource, memory}, {resource, disk}. + %% More reasons can be added in the future. + blocked_by, + %% true if received any publishes, false otherwise + %% note that this will also be true when connection is + %% already blocked + should_block, + %% true if we had we sent a connection.blocked, + %% false otherwise + connection_blocked_message_sent }). -define(STATISTICS_KEYS, [pid, recv_oct, recv_cnt, send_oct, send_cnt, @@ -171,7 +178,6 @@ -define(IS_RUNNING(State), (State#v1.connection_state =:= running orelse - State#v1.connection_state =:= blocking orelse State#v1.connection_state =:= blocked)). -define(IS_STOPPING(State), @@ -377,9 +383,11 @@ start_connection(Parent, HelperSup, Deb, Sock) -> channel_sup_sup_pid = none, channel_count = 0, throttle = #throttle{ - alarmed_by = [], - last_blocked_by = none, - last_blocked_at = never}}, + last_blocked_at = never, + should_block = false, + blocked_by = sets:new(), + connection_blocked_message_sent = false + }}, try run({?MODULE, recvloop, [Deb, [], 0, switch_callback(rabbit_event:init_stats_timer( @@ -519,19 +527,13 @@ stop(Reason, State) -> throw({inet_error, Reason}). handle_other({conserve_resources, Source, Conserve}, - State = #v1{throttle = Throttle = #throttle{alarmed_by = CR}}) -> - CR1 = case Conserve of - true -> lists:usort([Source | CR]); - false -> CR -- [Source] + State = #v1{throttle = Throttle = #throttle{blocked_by = Blockers}}) -> + Resource = {resource, Source}, + Blockers1 = case Conserve of + true -> sets:add_element(Resource, Blockers); + false -> sets:del_element(Resource, Blockers) end, - State1 = control_throttle( - State#v1{throttle = Throttle#throttle{alarmed_by = CR1}}), - case {blocked_by_alarm(State), blocked_by_alarm(State1)} of - {false, true} -> ok = send_blocked(State1); - {true, false} -> ok = send_unblocked(State1); - {_, _} -> ok - end, - State1; + control_throttle(State#v1{throttle = Throttle#throttle{blocked_by = Blockers1}}); handle_other({channel_closing, ChPid}, State) -> ok = rabbit_channel:ready_for_close(ChPid), {_, State1} = channel_cleanup(ChPid, State), @@ -618,52 +620,12 @@ terminate(Explanation, State) when ?IS_RUNNING(State) -> terminate(_Explanation, State) -> {force, State}. -control_throttle(State = #v1{connection_state = CS, throttle = Throttle}) -> - IsThrottled = ((Throttle#throttle.alarmed_by =/= []) orelse - credit_flow:blocked()), - case {CS, IsThrottled} of - {running, true} -> State#v1{connection_state = blocking}; - {blocking, false} -> State#v1{connection_state = running}; - {blocked, false} -> ok = rabbit_heartbeat:resume_monitor( - State#v1.heartbeater), - State#v1{connection_state = running}; - {blocked, true} -> State#v1{throttle = update_last_blocked_by( - Throttle)}; - {_, _} -> State - end. - -maybe_block(State = #v1{connection_state = blocking, - throttle = Throttle}) -> - ok = rabbit_heartbeat:pause_monitor(State#v1.heartbeater), - State1 = State#v1{connection_state = blocked, - throttle = update_last_blocked_by( - Throttle#throttle{ - last_blocked_at = - time_compat:monotonic_time()})}, - case {blocked_by_alarm(State), blocked_by_alarm(State1)} of - {false, true} -> ok = send_blocked(State1); - {_, _} -> ok - end, - State1; -maybe_block(State) -> - State. - - -blocked_by_alarm(#v1{connection_state = blocked, - throttle = #throttle{alarmed_by = CR}}) - when CR =/= [] -> - true; -blocked_by_alarm(#v1{}) -> - false. - -send_blocked(#v1{throttle = #throttle{alarmed_by = CR}, - connection = #connection{protocol = Protocol, +send_blocked(#v1{connection = #connection{protocol = Protocol, capabilities = Capabilities}, - sock = Sock}) -> + sock = Sock}, Reason) -> case rabbit_misc:table_lookup(Capabilities, <<"connection.blocked">>) of {bool, true} -> - RStr = string:join([atom_to_list(A) || A <- CR], " & "), - Reason = list_to_binary(rabbit_misc:format("low on ~s", [RStr])), + ok = send_on_channel0(Sock, #'connection.blocked'{reason = Reason}, Protocol); _ -> @@ -680,11 +642,6 @@ send_unblocked(#v1{connection = #connection{protocol = Protocol, ok end. -update_last_blocked_by(Throttle = #throttle{alarmed_by = []}) -> - Throttle#throttle{last_blocked_by = flow}; -update_last_blocked_by(Throttle) -> - Throttle#throttle{last_blocked_by = resource}. - %%-------------------------------------------------------------------------- %% error handling / termination @@ -999,9 +956,9 @@ post_process_frame({method, 'channel.close_ok', _}, ChPid, State) -> %% since we cannot possibly be in the 'closing' state. control_throttle(State1); post_process_frame({content_header, _, _, _, _}, _ChPid, State) -> - maybe_block(State); + publish_received(State); post_process_frame({content_body, _}, _ChPid, State) -> - maybe_block(State); + publish_received(State); post_process_frame(_Frame, _ChPid, State) -> State. @@ -1205,8 +1162,11 @@ handle_method0(#'connection.open'{virtual_host = VHostPath}, ok = rabbit_access_control:check_vhost_access(User, VHostPath, Sock), NewConnection = Connection#connection{vhost = VHostPath}, ok = send_on_channel0(Sock, #'connection.open_ok'{}, Protocol), - Conserve = rabbit_alarm:register(self(), {?MODULE, conserve_resources, []}), - Throttle1 = Throttle#throttle{alarmed_by = Conserve}, + + Alarms = rabbit_alarm:register(self(), {?MODULE, conserve_resources, []}), + BlockedBy = sets:from_list([{resource, Alarm} || Alarm <- Alarms]), + Throttle1 = Throttle#throttle{blocked_by = BlockedBy}, + {ok, ChannelSupSupPid} = rabbit_connection_helper_sup:start_channel_sup_sup(SupPid), State1 = control_throttle( @@ -1394,18 +1354,29 @@ i(peer_cert_subject, S) -> cert_info(fun rabbit_ssl:peer_cert_subject/1, S); i(peer_cert_validity, S) -> cert_info(fun rabbit_ssl:peer_cert_validity/1, S); i(channels, #v1{channel_count = ChannelCount}) -> ChannelCount; i(state, #v1{connection_state = ConnectionState, - throttle = #throttle{alarmed_by = Alarms, - last_blocked_by = WasBlockedBy, - last_blocked_at = T}}) -> - case Alarms =:= [] andalso %% not throttled by resource alarms - (credit_flow:blocked() %% throttled by flow now - orelse %% throttled by flow recently - (WasBlockedBy =:= flow andalso T =/= never andalso + throttle = #throttle{blocked_by = Reasons, + last_blocked_at = T} = Throttle}) -> + %% not throttled by resource or other longer-term reasons + %% TODO: come up with a sensible function name + case sets:size(sets:del_element(flow, Reasons)) =:= 0 andalso + (credit_flow:blocked() %% throttled by flow now + orelse %% throttled by flow recently + (is_blocked_by_flow(Throttle) andalso T =/= never andalso time_compat:convert_time_unit(time_compat:monotonic_time() - T, native, micro_seconds) < 5000000)) of true -> flow; - false -> ConnectionState + false -> + case {has_reasons_to_block(Throttle), ConnectionState} of + %% blocked + {_, blocked} -> blocked; + %% not yet blocked (there were no publishes) + {true, running} -> blocking; + %% not blocked + {false, _} -> ConnectionState; + %% catch all to be defensive + _ -> ConnectionState + end end; i(Item, #v1{connection = Conn}) -> ic(Item, Conn). @@ -1505,3 +1476,118 @@ send_error_on_channel0_and_close(Channel, Protocol, Reason, State) -> State1 = close_connection(terminate_channels(State)), ok = send_on_channel0(State#v1.sock, CloseMethod, Protocol), State1. + +%% +%% Publisher throttling +%% + +blocked_by_message(#throttle{blocked_by = Reasons}) -> + %% we don't want to report internal flow as a reason here since + %% it is entirely transient + Reasons1 = sets:del_element(flow, Reasons), + RStr = string:join([format_blocked_by(R) || R <- sets:to_list(Reasons1)], " & "), + list_to_binary(rabbit_misc:format("low on ~s", [RStr])). + +format_blocked_by({resource, memory}) -> "memory"; +format_blocked_by({resource, disk}) -> "disk"; +format_blocked_by({resource, disc}) -> "disk". + +update_last_blocked_at(Throttle) -> + Throttle#throttle{last_blocked_at = time_compat:monotonic_time()}. + +connection_blocked_message_sent( + #throttle{connection_blocked_message_sent = BS}) -> BS. + +should_send_blocked(Throttle = #throttle{blocked_by = Reasons}) -> + should_block(Throttle) + andalso + sets:size(sets:del_element(flow, Reasons)) =/= 0 + andalso + not connection_blocked_message_sent(Throttle). + +should_send_unblocked(Throttle = #throttle{blocked_by = Reasons}) -> + connection_blocked_message_sent(Throttle) + andalso + sets:size(sets:del_element(flow, Reasons)) == 0. + +%% Returns true if we have a reason to block +%% this connection. +has_reasons_to_block(#throttle{blocked_by = Reasons}) -> + sets:size(Reasons) > 0. + +is_blocked_by_flow(#throttle{blocked_by = Reasons}) -> + sets:is_element(flow, Reasons). + +should_block(#throttle{should_block = Val}) -> Val. + +should_block_connection(Throttle) -> + should_block(Throttle) andalso has_reasons_to_block(Throttle). + +should_unblock_connection(Throttle) -> + not should_block_connection(Throttle). + +maybe_block(State = #v1{connection_state = CS, throttle = Throttle}) -> + case should_block_connection(Throttle) of + true -> + State1 = State#v1{connection_state = blocked, + throttle = update_last_blocked_at(Throttle)}, + case CS of + running -> + ok = rabbit_heartbeat:pause_monitor(State#v1.heartbeater); + _ -> ok + end, + maybe_send_blocked_or_unblocked(State1); + false -> State + end. + +maybe_unblock(State = #v1{throttle = Throttle}) -> + case should_unblock_connection(Throttle) of + true -> + ok = rabbit_heartbeat:resume_monitor(State#v1.heartbeater), + State1 = State#v1{connection_state = running, + throttle = Throttle#throttle{should_block = false}}, + maybe_send_unblocked(State1); + false -> State + end. + +maybe_send_unblocked(State = #v1{throttle = Throttle}) -> + case should_send_unblocked(Throttle) of + true -> + ok = send_unblocked(State), + State#v1{throttle = + Throttle#throttle{connection_blocked_message_sent = false}}; + false -> State + end. + +maybe_send_blocked_or_unblocked(State = #v1{throttle = Throttle}) -> + case should_send_blocked(Throttle) of + true -> + ok = send_blocked(State, blocked_by_message(Throttle)), + State#v1{throttle = + Throttle#throttle{connection_blocked_message_sent = true}}; + false -> maybe_send_unblocked(State) + end. + +publish_received(State = #v1{throttle = Throttle}) -> + case has_reasons_to_block(Throttle) of + false -> State; + true -> + Throttle1 = Throttle#throttle{should_block = true}, + maybe_block(State#v1{throttle = Throttle1}) + end. + +control_throttle(State = #v1{connection_state = CS, + throttle = #throttle{blocked_by = Reasons} = Throttle}) -> + Throttle1 = case credit_flow:blocked() of + true -> + Throttle#throttle{blocked_by = sets:add_element(flow, Reasons)}; + false -> + Throttle#throttle{blocked_by = sets:del_element(flow, Reasons)} + end, + State1 = State#v1{throttle = Throttle1}, + case CS of + running -> maybe_block(State1); + %% unblock or re-enable blocking + blocked -> maybe_block(maybe_unblock(State1)); + _ -> State1 + end. From f03d2b0d52c46cd063483f396789cc92a2e8ea88 Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Fri, 18 Mar 2016 20:10:40 +0300 Subject: [PATCH 39/45] Fix listing of down queues `rabbit_amqqueue:info_down/2` was erroneously used insted of `rabbit_amqqueue:info_down/3`. Fixes https://github.com/rabbitmq/rabbitmq-server/issues/696 And thanks to this issue it is now preemptively fixed in https://github.com/rabbitmq/rabbitmq-server/pull/683 ) --- src/rabbit_amqqueue.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index c4975b5c..fe24d125 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -629,7 +629,7 @@ info_all(VHostPath, Items, Ref, AggregatorPid) -> AggregatorPid, Ref, fun(Q) -> info(Q, Items) end, list(VHostPath), continue), rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> info_down(Q, Items) end, + AggregatorPid, Ref, fun(Q) -> info_down(Q, Items, down) end, list_down(VHostPath)). force_event_refresh(Ref) -> From e1ac8454cc10ce7c5ded7662ddfa6bf7bd0131f1 Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Wed, 23 Mar 2016 18:13:16 +0300 Subject: [PATCH 40/45] Fix `rabbitmqctl list_consumers` It makes no sense to concatenate information about several consumers into single proplist. `rabbit_amqqueue:consumers_all/1` is already doing `lists:append/1`, so it is already capable of handling zero or more consumers reported. And for `rabbitmqctl list_consumers` we need to support printing lists-of-lists, for the same purpose of reporting zero or more consumers. Another part for https://github.com/rabbitmq/rabbitmq-server/issues/701 --- src/rabbit_amqqueue.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index fe24d125..761c0976 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -658,11 +658,10 @@ consumers_all(VHostPath, Ref, AggregatorPid) -> list(VHostPath)). get_queue_consumer_info(Q, ConsumerInfoKeys) -> - lists:flatten( - [lists:zip(ConsumerInfoKeys, - [Q#amqqueue.name, ChPid, CTag, - AckRequired, Prefetch, Args]) || - {ChPid, CTag, AckRequired, Prefetch, Args} <- consumers(Q)]). + [lists:zip(ConsumerInfoKeys, + [Q#amqqueue.name, ChPid, CTag, + AckRequired, Prefetch, Args]) || + {ChPid, CTag, AckRequired, Prefetch, Args} <- consumers(Q)]. stat(#amqqueue{pid = QPid}) -> delegate:call(QPid, stat). From 9c705faa47729e2eebbe2901faf3919980fdbcc7 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Wed, 23 Mar 2016 18:52:05 +0300 Subject: [PATCH 41/45] Commit rabbitmq-components.mk changes --- mk/rabbitmq-components.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mk/rabbitmq-components.mk b/mk/rabbitmq-components.mk index eed26fda..b200585b 100644 --- a/mk/rabbitmq-components.mk +++ b/mk/rabbitmq-components.mk @@ -66,6 +66,8 @@ dep_rabbitmq_test = git_rmq rabbitmq-test $(current_rmq_ref) dep_rabbitmq_web_dispatch = git_rmq rabbitmq-web-dispatch $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_web_stomp = git_rmq rabbitmq-web-stomp $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_web_stomp_examples = git_rmq rabbitmq-web-stomp-examples $(current_rmq_ref) $(base_rmq_ref) master +dep_rabbitmq_web_mqtt = git_rmq rabbitmq-web-mqtt $(current_rmq_ref) $(base_rmq_ref) master +dep_rabbitmq_web_mqtt_examples = git_rmq rabbitmq-web-mqtt-examples $(current_rmq_ref) $(base_rmq_ref) master dep_rabbitmq_website = git_rmq rabbitmq-website $(current_rmq_ref) $(base_rmq_ref) live master dep_sockjs = git_rmq sockjs-erlang $(current_rmq_ref) $(base_rmq_ref) master dep_toke = git_rmq toke $(current_rmq_ref) $(base_rmq_ref) master @@ -117,6 +119,8 @@ RABBITMQ_COMPONENTS = amqp_client \ rabbitmq_top \ rabbitmq_tracing \ rabbitmq_web_dispatch \ + rabbitmq_web_mqtt \ + rabbitmq_web_mqtt_examples \ rabbitmq_web_stomp \ rabbitmq_web_stomp_examples \ rabbitmq_website From f9cb3eb84493ce1fb404f88c43394bcf5678337c Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Thu, 24 Mar 2016 15:27:51 +0300 Subject: [PATCH 42/45] Add support for listing up/down queues Part of https://github.com/rabbitmq/rabbitmq-server/issues/688 --- src/rabbit_amqqueue.erl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index fe24d125..5287b465 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -25,7 +25,7 @@ check_exclusive_access/2, with_exclusive_access_or_die/3, stat/1, deliver/2, requeue/3, ack/3, reject/4]). -export([list/0, list/1, info_keys/0, info/1, info/2, info_all/1, info_all/2, - info_all/4]). + info_all/6]). -export([list_down/1]). -export([force_event_refresh/1, notify_policy_changed/1]). -export([consumers/1, consumers_all/1, consumers_all/3, consumer_info_keys/0]). @@ -624,13 +624,16 @@ info_all(VHostPath, Items) -> map(list(VHostPath), fun (Q) -> info(Q, Items) end) ++ map(list_down(VHostPath), fun (Q) -> info_down(Q, Items, down) end). -info_all(VHostPath, Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> info(Q, Items) end, list(VHostPath), - continue), - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> info_down(Q, Items, down) end, - list_down(VHostPath)). +info_all(VHostPath, Items, NeedOnline, NeedOffline, Ref, AggregatorPid) -> + NeedOnline andalso rabbit_control_misc:emitting_map_with_exit_handler( + AggregatorPid, Ref, fun(Q) -> info(Q, Items) end, list(VHostPath), + continue), + NeedOffline andalso rabbit_control_misc:emitting_map_with_exit_handler( + AggregatorPid, Ref, fun(Q) -> info_down(Q, Items, down) end, + list_down(VHostPath), + continue), + %% Previous maps are incomplete, finalize emission + rabbit_control_misc:emitting_map(AggregatorPid, Ref, fun(_) -> no_op end, []). force_event_refresh(Ref) -> [gen_server2:cast(Q#amqqueue.pid, From cba2f79fb918843fd0a84b4f725091ab80e874d7 Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Wed, 23 Mar 2016 18:13:16 +0300 Subject: [PATCH 43/45] Fix `rabbitmqctl list_consumers` It makes no sense to concatenate information about several consumers into single proplist. `rabbit_amqqueue:consumers_all/1` is already doing `lists:append/1`, so it is already capable of handling zero or more consumers reported. And for `rabbitmqctl list_consumers` we need to support printing lists-of-lists, for the same purpose of reporting zero or more consumers. Another part for https://github.com/rabbitmq/rabbitmq-server/issues/701 --- src/rabbit_amqqueue.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl index fe24d125..761c0976 100644 --- a/src/rabbit_amqqueue.erl +++ b/src/rabbit_amqqueue.erl @@ -658,11 +658,10 @@ consumers_all(VHostPath, Ref, AggregatorPid) -> list(VHostPath)). get_queue_consumer_info(Q, ConsumerInfoKeys) -> - lists:flatten( - [lists:zip(ConsumerInfoKeys, - [Q#amqqueue.name, ChPid, CTag, - AckRequired, Prefetch, Args]) || - {ChPid, CTag, AckRequired, Prefetch, Args} <- consumers(Q)]). + [lists:zip(ConsumerInfoKeys, + [Q#amqqueue.name, ChPid, CTag, + AckRequired, Prefetch, Args]) || + {ChPid, CTag, AckRequired, Prefetch, Args} <- consumers(Q)]. stat(#amqqueue{pid = QPid}) -> delegate:call(QPid, stat). From 12a0fe0018f0ef0199fb322c38905d03cb0206b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Tue, 29 Mar 2016 15:16:43 +0200 Subject: [PATCH 44/45] Travis CI: Fix typo in comment --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9986cece..3fefdff2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ otp_release: # # FIXME: There is still one problem: for builds triggered by a pull # request, $TRAVIS_BRANCH contains the target branch name, not the -# soruce branch name. Therefore, we can't rely on automatic checkout +# source branch name. Therefore, we can't rely on automatic checkout # of corresponding branches in dependencies. For instance, if the pull # request comes from a branch "rabbitmq-server-123", based on "stable", # then this command will checkout "stable" and we won't try to checkout From 6e33d6d945af3989e611e8ab00434f4e35d05a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20P=C3=A9dron?= Date: Wed, 30 Mar 2016 19:38:12 +0200 Subject: [PATCH 45/45] Move test TLS certificates creation to rabbitmq-common In the future, this will avoid a test-only dependency to rabbitmq-test just for that. In particular, it's useful in the context of rabbitmq-erlang-client which is also a dependency of rabbitmq-test currently (circular dependency). --- tools/tls-certs/Makefile | 67 +++++++++++++++++++++++++++++++++++++ tools/tls-certs/openssl.cnf | 54 ++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 tools/tls-certs/Makefile create mode 100644 tools/tls-certs/openssl.cnf diff --git a/tools/tls-certs/Makefile b/tools/tls-certs/Makefile new file mode 100644 index 00000000..77995875 --- /dev/null +++ b/tools/tls-certs/Makefile @@ -0,0 +1,67 @@ +ifndef DIR +$(error DIR must be specified) +endif + +PASSWORD ?= changeme + +# Verbosity. + +V ?= 0 + +verbose_0 = @ +verbose_2 = set -x; +verbose = $(verbose_$(V)) + +gen_verbose_0 = @echo " GEN " $@; +gen_verbose_2 = set -x; +gen_verbose = $(gen_verbose_$(V)) + +openssl_output_0 = 2>/dev/null +openssl_output = $(openssl_output_$(V)) + +.PRECIOUS: %/testca/cacert.pem +.PHONY: all testca server client clean + +all: server client + @: + +testca: $(DIR)/testca/cacert.pem + +server: TARGET = server +server: $(DIR)/server/cert.pem + @: + +client: TARGET = client +client: $(DIR)/client/cert.pem + @: + +$(DIR)/testca/cacert.pem: + $(gen_verbose) mkdir -p $(dir $@) + $(verbose) { ( cd $(dir $@) && \ + mkdir -p certs private && \ + chmod 700 private && \ + echo 01 > serial && \ + :> index.txt && \ + openssl req -x509 -config $(CURDIR)/openssl.cnf -newkey rsa:2048 -days 365 \ + -out cacert.pem -outform PEM -subj /CN=MyTestCA/L=$$$$/ -nodes && \ + openssl x509 -in cacert.pem -out cacert.cer -outform DER ) $(openssl_output) \ + || (rm -rf $(dir $@) && false); } + +$(DIR)/%/cert.pem: $(DIR)/testca/cacert.pem + $(gen_verbose) mkdir -p $(DIR)/$(TARGET) + $(verbose) { ( cd $(DIR)/$(TARGET) && \ + openssl genrsa -out key.pem 2048 &&\ + openssl req -new -key key.pem -out req.pem -outform PEM\ + -subj /CN=$$(hostname)/O=$(TARGET)/L=$$$$/ -nodes &&\ + cd ../testca && \ + openssl ca -config $(CURDIR)/openssl.cnf -in ../$(TARGET)/req.pem -out \ + ../$(TARGET)/cert.pem -notext -batch -extensions \ + $(TARGET)_ca_extensions && \ + cd ../$(TARGET) && \ + openssl pkcs12 -export -out keycert.p12 -in cert.pem -inkey key.pem \ + -passout pass:$(PASSWORD) ) $(openssl_output) || (rm -rf $(DIR)/$(TARGET) && false); } + +clean: + rm -rf $(DIR)/testca + rm -rf $(DIR)/server + rm -rf $(DIR)/client diff --git a/tools/tls-certs/openssl.cnf b/tools/tls-certs/openssl.cnf new file mode 100644 index 00000000..93ffb2fd --- /dev/null +++ b/tools/tls-certs/openssl.cnf @@ -0,0 +1,54 @@ +[ ca ] +default_ca = testca + +[ testca ] +dir = . +certificate = $dir/cacert.pem +database = $dir/index.txt +new_certs_dir = $dir/certs +private_key = $dir/private/cakey.pem +serial = $dir/serial + +default_crl_days = 7 +default_days = 365 +default_md = sha1 + +policy = testca_policy +x509_extensions = certificate_extensions + +[ testca_policy ] +commonName = supplied +stateOrProvinceName = optional +countryName = optional +emailAddress = optional +organizationName = optional +organizationalUnitName = optional +domainComponent = optional + +[ certificate_extensions ] +basicConstraints = CA:false + +[ req ] +default_bits = 2048 +default_keyfile = ./private/cakey.pem +default_md = sha1 +prompt = yes +distinguished_name = root_ca_distinguished_name +x509_extensions = root_ca_extensions + +[ root_ca_distinguished_name ] +commonName = hostname + +[ root_ca_extensions ] +basicConstraints = CA:true +keyUsage = keyCertSign, cRLSign + +[ client_ca_extensions ] +basicConstraints = CA:false +keyUsage = digitalSignature +extendedKeyUsage = 1.3.6.1.5.5.7.3.2 + +[ server_ca_extensions ] +basicConstraints = CA:false +keyUsage = keyEncipherment +extendedKeyUsage = 1.3.6.1.5.5.7.3.1