Merge branch 'stable'

Conflicts:
	.travis.yml
This commit is contained in:
Michael Klishin 2017-02-10 20:34:53 +03:00
commit f9d5104344
18 changed files with 505 additions and 29 deletions

View File

@ -12,8 +12,9 @@ endef
DEPS = rabbit_common rabbit
LOCAL_DEPS += ssl crypto public_key
## We need the Cowboy's test utilities
TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client ct_helper
TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client ct_helper trust_store_http
dep_ct_helper = git https://github.com/extend/ct_helper.git master
dep_trust_store_http = git https://github.com/hairyhum/trust_store_http.git
DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk

View File

@ -12,12 +12,19 @@ through various TLS socket options, namely the `ca_certs` and
There is no convenient means with which to change it in realtime, that
is, without making configuration changes to TLS listening sockets.
This plugin maintains a list of trusted .PEM formatted TLS (x509) certificates in a given
directory, refreshing at configurable intervals, or when `rabbitmqctl
This plugin maintains a list of trusted .PEM formatted TLS (x509) certificates,
refreshing at configurable intervals, or when `rabbitmqctl
eval 'rabbit_trust_store:refresh().'` is invoked. Said certificates are then used
to verify inbound TLS connections for the entire RabbitMQ node (all plugins and protocols).
The list is node-local.
Certificates can be loaded from different sources (e.g. filesystem, HTTP server)
Sources are loaded using "providers" - erlang modules, implementing `rabbit_trust_store_certificate_provider`
behaviour.
The default provider is `rabbit_trust_store_file_provider`, which will load certificates
from a configured local filesystem directory.
## RabbitMQ Version Requirements
This plugin requires RabbitMQ `3.6.1` or later.
@ -33,6 +40,8 @@ Please consult the docs on [how to install RabbitMQ plugins](http://www.rabbitmq
## Usage
### Filesystem provider
Configure the trust store with a directory of whitelisted certificates
and a refresh interval:
@ -47,14 +56,14 @@ Setting `refresh_interval` to `0` seconds will disable automatic refresh.
Certificates are distinguished by their **filenames** and file modification time.
### Installing a Certificate
#### Installing a Certificate
Write a `PEM` formatted certificate file to the configured directory
to whitelist it. This contains all the necessary information to
authorize a client which presents the very same certificate to the
server.
### Removing a Certificate
#### Removing a Certificate
Delete the certificate file from the configured directory to remove it
from the whitelist.
@ -64,6 +73,50 @@ make it seem as if a removed certificate is still active. Disabling session cach
in the broker by setting the `reuse_sessions` ssl option to `false` can be done if
timely certificate removal is important.
### HTTP provider
HTTP provider loads certificates via HTTP(S) from remote server.
The server should have following API:
- `GET <root>` - list certificates in JSON format: `{"certificates": [{"id": <id>, "url": <url>}, ...]}`
- `GET <root>/<url>` - download PEM encoded certificate.
Where `<root>` is a configured certificate path, `<id>` - unique certificate identifier,
`<url>` - relative certificate path to load it from server.
Configuration of the HTTP provider:
```
{rabbitmq_trust_store,
[{providers, [rabbit_trust_store_http_provider]},
{url, "http://example.cert.url/path"},
{refresh_interval, {seconds, 30}}
]}.
```
You can specify TLS options if you use HTTPS:
```
{rabbitmq_trust_store,
[{providers, [rabbit_trust_store_http_provider]},
{url, "https://example.secure.cert.url/path"},
{refresh_interval, {seconds, 30}},
{ssl_options, [{certfile, "/client/cert.pem"},
{keyfile, "/client/key.pem"},
{cacertfile, "/ca/cert.pem"}
]}
]}.
```
HTTP provider uses `If-Modified-Since` during list request header to avoid updating
unchanged list of certificates.
#### Example
`examples/rabbitmq_trust_store_django` is an example Django application, which serves
certificates from a directory.
### Listing certificates

View File

@ -0,0 +1,3 @@
*.pyc
*.sqlite3
certs/*

View File

@ -0,0 +1,49 @@
# RabbitMQ trust store HTTP server example
`rabbitmq_trust_store_django` is a very minimalistic [Django](https://www.djangoproject.com/) 1.10+ application
that rabbitmq-trust-store's `rabbit_trust_store_http_provider` can use as a source of certificates.
The project serves PEM encoded CA certificate files from the `certs` directory.
It's really not designed to be anything other than an example.
## Running the Example
1. Put certificates that should be trusted in PEM format into the `certs` directory.
2. Run `python manage.py runserver` to launch it after [installing Django](https://docs.djangoproject.com/en/1.10/topics/install/).
3. Configure RabbitMQ trust store to use `rabbit_trust_store_http_provider`.
Below is a very minimalistic example that assumes the example is available
on the local machine via HTTP:
```
{rabbitmq_trust_store,
[{providers, [rabbit_trust_store_http_provider]},
{url, "http://127.0.0.1:8000/"},
{refresh_interval, {seconds, 30}}
]}.
```
You can specify TLS options if you use HTTPS:
```
{rabbitmq_trust_store,
[{providers, [rabbit_trust_store_http_provider]},
{url, "https://secure.store.example.local:8000/"},
{refresh_interval, {seconds, 30}},
{ssl_options, [{certfile, "/client/cert.pem"},
{keyfile, "/client/key.pem"},
{cacertfile, "/ca/cert.pem"}
]}
]}.
```
## HTTP API Endpoints
This project will serve static files from `certs` directory and
will list them in JSON format described in [rabbitmq-trust-store](https://github.com/rabbitmq/rabbitmq-trust-store/)
If you're not familiar with Django, urls.py and auth/views.py may be
most illuminating.

View File

@ -0,0 +1,10 @@
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rabbitmq_trust_store_django.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)

View File

@ -0,0 +1,125 @@
"""
Django settings for rabbitmq_trust_store_django project.
Generated by 'django-admin startproject' using Django 1.9.4.
For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'y3ws-)+baz%a4^p(2@+ell%&bj-q2q^f2&do)c2-feo#a4*x0t'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
]
MIDDLEWARE_CLASSES = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'rabbitmq_trust_store_django.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'rabbitmq_trust_store_django.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/
STATIC_URL = "/certs/"
STATICFILES_DIRS = [
os.path.join(BASE_DIR, "certs")
]

View File

@ -0,0 +1,7 @@
from __future__ import unicode_literals
from django.apps import AppConfig
class TrustStoreConfig(AppConfig):
name = 'trust_store'

View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

View File

@ -0,0 +1,7 @@
from django.conf.urls import url
from . import views
urlpatterns = [
url(r'^$', views.index, name='index'),
]

View File

@ -0,0 +1,41 @@
from django.shortcuts import render
from django.http import HttpResponse, JsonResponse
import os
from django.conf import settings
from django.views.decorators.http import last_modified
from datetime import datetime
def latest_dir_change(request):
timestamp = os.stat(cert_directory()).st_mtime
return datetime.fromtimestamp(timestamp)
@last_modified(latest_dir_change)
def index(request):
request.META
directory = cert_directory()
certs = {'certificates': [file_object(file) for file in pem_files(directory)]}
return JsonResponse(certs)
def cert_directory():
return os.path.join(settings.BASE_DIR, "certs")
def pem_files(directory):
files = os.listdir(directory)
return [os.path.join(directory, file) for file in files if is_pem(file)]
def is_pem(file):
return 'pem' == os.path.splitext(file)[1][1:]
def file_object(file):
return {'id': file_id(file), 'path': path_for_file(file)}
def file_id(file):
mtime = str(int(os.stat(file).st_mtime))
basename = os.path.basename(file)
return basename + ':' + mtime
def path_for_file(file):
basename = os.path.basename(file)
return "/certs/" + basename

View File

@ -0,0 +1,21 @@
"""rabbitmq_trust_store_django URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.conf.urls import url, include
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import include, url
from django.contrib import admin
urlpatterns = [
url(r'', include('rabbitmq_trust_store_django.trust_store.urls')),
]

View File

@ -0,0 +1,16 @@
"""
WSGI config for rabbitmq_trust_store_django project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rabbitmq_trust_store_django.settings")
application = get_wsgi_application()

View File

@ -229,7 +229,8 @@ refresh_provider_certs(Provider, Config, ProviderState) ->
{error, Reason} ->
rabbit_log:error("Unable to load certificate list for provider ~p,"
" reason: ~p~n",
[Provider, Reason])
[Provider, Reason]),
ProviderState
end.
list_certs(Provider, Config, nostate) ->

View File

@ -3,13 +3,14 @@
-include_lib("public_key/include/public_key.hrl").
-callback list_certs(Config)
-> no_change | {ok, [{CertId, Attributes}]}
-> no_change | {ok, [{CertId, Attributes}], ProviderState}
when Config :: list(),
ProviderState :: term(),
CertId :: term(),
Attributes :: list().
-callback list_certs(Config, ProviderState)
-> no_change | {ok, [{CertId, Attributes}]}
-> no_change | {ok, [{CertId, Attributes}], ProviderState}
when Config :: list(),
ProviderState :: term(),
CertId :: term(),

View File

@ -0,0 +1,100 @@
-module(rabbit_trust_store_http_provider).
-include_lib("public_key/include/public_key.hrl").
-behaviour(rabbit_trust_store_certificate_provider).
-export([list_certs/1, list_certs/2, load_cert/3]).
-record(http_state,{
url :: string(),
http_options :: list(),
headers :: httpc:headers()
}).
list_certs(Config) ->
init(),
State = init_state(Config),
list_certs(Config, State).
list_certs(_, #http_state{url = Url,
http_options = HttpOptions,
headers = Headers} = State) ->
Res = httpc:request(get, {Url, Headers}, HttpOptions, [{body_format, binary}]),
case Res of
{ok, {{_, 200, _}, RespHeaders, Body}} ->
Certs = decode_cert_list(Body),
NewState = new_state(RespHeaders, State),
{ok, Certs, NewState};
{ok, {{_,304, _}, _, _}} -> no_change;
{ok, {{_,Code,_}, _, Body}} -> {error, {http_error, Code, Body}};
{error, Reason} -> {error, Reason}
end.
load_cert(_, Attributes, Config) ->
CertPath = proplists:get_value(path, Attributes),
#http_state{url = BaseUrl,
http_options = HttpOptions,
headers = Headers} = init_state(Config),
Url = join_url(BaseUrl, CertPath),
Res = httpc:request(get,
{Url, Headers},
HttpOptions,
[{body_format, binary}, {full_result, false}]),
case Res of
{ok, {200, Body}} ->
[{'Certificate', Cert, not_encrypted}] = public_key:pem_decode(Body),
{ok, Cert};
{ok, {Code, Body}} -> {error, {http_error, Code, Body}};
{error, Reason} -> {error, Reason}
end.
join_url(BaseUrl, CertPath) ->
string:strip(rabbit_data_coercion:to_list(BaseUrl), right, $/)
++ "/" ++
string:strip(rabbit_data_coercion:to_list(CertPath), left, $/).
init() ->
inets:start(),
ssl:start().
init_state(Config) ->
Url = proplists:get_value(url, Config),
Headers = proplists:get_value(http_headers, Config, []),
HttpOptions = case proplists:get_value(ssl_options, Config) of
undefined -> [];
SslOpts -> [{ssl, SslOpts}]
end,
#http_state{url = Url, http_options = HttpOptions, headers = Headers}.
decode_cert_list(Body) ->
{ok, Struct} = rabbit_misc:json_decode(Body),
[{<<"certificates">>, Certs}] = rabbit_misc:json_to_term(Struct),
lists:map(
fun(Cert) ->
Path = proplists:get_value(<<"path">>, Cert),
CertId = proplists:get_value(<<"id">>, Cert),
{CertId, [{path, Path}]}
end,
Certs).
new_state(RespHeaders, #http_state{headers = Headers} = State) ->
LastModified = proplists:get_value("Last-Modified",
RespHeaders,
proplists:get_value("last-modified",
RespHeaders,
undefined)),
case LastModified of
undefined -> State;
Value ->
NewHeaders = lists:ukeymerge(1, Headers,
[{"If-Modified-Since", Value}]),
State#http_state{headers = NewHeaders}
end.

View File

@ -8,31 +8,35 @@
-define(SERVER_REJECT_CLIENT, {tls_alert, "unknown ca"}).
all() ->
[
{group, http_provider_tests},
{group, file_provider_tests}
].
groups() ->
CommonTests = [
validate_chain,
validate_longer_chain,
validate_chain_without_whitelisted,
whitelisted_certificate_accepted_from_AMQP_client_regardless_of_validation_to_root,
removed_certificate_denied_from_AMQP_client,
installed_certificate_accepted_from_AMQP_client,
whitelist_directory_DELTA,
replaced_whitelisted_certificate_should_be_accepted,
ensure_configuration_using_binary_strings_is_handled,
ignore_corrupt_cert,
ignore_same_cert_with_different_name,
list],
[
{file_provider_tests, [], [
library,
invasive_SSL_option_change,
validation_success_for_AMQP_client,
validation_failure_for_AMQP_client,
validate_chain,
validate_longer_chain,
validate_chain_without_whitelisted,
whitelisted_certificate_accepted_from_AMQP_client_regardless_of_validation_to_root,
removed_certificate_denied_from_AMQP_client,
installed_certificate_accepted_from_AMQP_client,
whitelist_directory_DELTA,
replaced_whitelisted_certificate_should_be_accepted,
ensure_configuration_using_binary_strings_is_handled,
ignore_corrupt_cert,
ignore_same_cert_with_different_name,
list,
disabled_provider_removes_certificates,
enabled_provider_adds_cerificates
]}
enabled_provider_adds_cerificates |
CommonTests
]},
{http_provider_tests, [], CommonTests}
].
suite() ->
@ -59,28 +63,62 @@ end_per_suite(Config) ->
init_per_group(file_provider_tests, Config) ->
WhitelistDir = filename:join([?config(rmq_certsdir, Config),
"trust_store",
"file_provider_tests"]),
ok = filelib:ensure_dir(WhitelistDir),
ok = file:make_dir(WhitelistDir),
Config1 = rabbit_ct_helpers:set_config(Config, {whitelist_dir, WhitelistDir}),
"trust_store", "file_provider_tests"]),
Config1 = init_whitelist_dir(Config, WhitelistDir),
ok = rabbit_ct_broker_helpers:rpc(Config, 0,
?MODULE, change_configuration,
[rabbitmq_trust_store, [{directory, WhitelistDir},
{refresh_interval, interval()},
{providers, [rabbit_trust_store_file_provider]}]]),
Config1.
Config1;
init_per_group(http_provider_tests, Config) ->
WhitelistDir = filename:join([?config(rmq_certsdir, Config),
"trust_store", "http_provider_tests"]),
Config1 = init_whitelist_dir(Config, WhitelistDir),
Config2 = init_provider_server(Config1, WhitelistDir),
Url = ?config(trust_store_server_url, Config2),
ok = rabbit_ct_broker_helpers:rpc(Config2, 0,
?MODULE, change_configuration,
[rabbitmq_trust_store, [{url, Url},
{refresh_interval, interval()},
{providers, [rabbit_trust_store_http_provider]}]]),
Config2.
init_provider_server(Config, WhitelistDir) ->
%% Assume we don't have more than 100 ports allocated for tests
PortBase = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_ports_base),
CertServerPort = PortBase + 100,
Url = "http://127.0.0.1:" ++ integer_to_list(CertServerPort) ++ "/",
application:load(trust_store_http),
ok = application:set_env(trust_store_http, directory, WhitelistDir),
ok = application:set_env(trust_store_http, port, CertServerPort),
ok = application:unset_env(trust_store_http, ssl_options),
application:ensure_all_started(trust_store_http),
rabbit_ct_helpers:set_config(Config, [{trust_store_server_port, CertServerPort},
{trust_store_server_url, Url}]).
end_per_group(http_provider_tests, Config) ->
application:stop(trust_store_http),
Config;
end_per_group(_, Config) ->
Config.
init_whitelist_dir(Config, WhitelistDir) ->
ok = filelib:ensure_dir(WhitelistDir),
ok = file:make_dir(WhitelistDir),
rabbit_ct_helpers:set_config(Config, {whitelist_dir, WhitelistDir}).
init_per_testcase(Testcase, Config) ->
WhitelistDir = ?config(whitelist_dir, Config),
ok = rabbit_file:recursive_delete([WhitelistDir]),
ok = file:make_dir(WhitelistDir),
ok = rabbit_ct_broker_helpers:rpc(Config, 0,
?MODULE, change_configuration,
[rabbitmq_trust_store, [{directory, WhitelistDir}]]),
[rabbitmq_trust_store, []]),
rabbit_ct_helpers:testcase_started(Config, Testcase).
end_per_testcase(Testcase, Config) ->