diff --git a/README.rst b/README.rst index 36bcdbe..1a73c6d 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,12 @@ To see full usage instructions for the ``start`` action, use ``-h``/``--help``: -h, --help show this help message and exit MapR arguments: + --kerberos If specified, enable Kerberos for the cluster (default: False) + --kerberos-principals If specified, a comma-separated list of Kerberos user principals + to create in KDC (default: None) + --kerberos-ticket-lifetime + If specified, the maximum time period in seconds for which + a ticket may be valid in the realm (default: 86400) --license-credentials credentials MapR license credentials to use in the format username:password (default: None) diff --git a/images/primary_node/Dockerfile b/images/primary_node/Dockerfile index c2c46a5..a958d27 100644 --- a/images/primary_node/Dockerfile +++ b/images/primary_node/Dockerfile @@ -27,6 +27,8 @@ RUN . /tmp/${REPO_URL_INSERT_SCRIPT} ${MAPR_VERSION} ${MEP_VERSION} RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key RUN yum -y install java-1.8.0-openjdk-devel \ + krb5-libs \ + krb5-workstation \ mapr-cldb \ mapr-core \ mapr-fileserver \ diff --git a/images/secondary_node/Dockerfile b/images/secondary_node/Dockerfile index 0c2b8b2..a83a46f 100644 --- a/images/secondary_node/Dockerfile +++ b/images/secondary_node/Dockerfile @@ -28,6 +28,8 @@ RUN . /tmp/${REPO_URL_INSERT_SCRIPT} ${MAPR_VERSION} ${MEP_VERSION} RUN rpm --import http://package.mapr.com/releases/pub/maprgpg.key RUN yum -y install java-1.8.0-openjdk-devel \ + krb5-libs \ + krb5-workstation \ mapr-core \ mapr-fileserver \ mapr-hbase-regionserver \ diff --git a/kerberos_helper.py b/kerberos_helper.py new file mode 100644 index 0000000..576eacc --- /dev/null +++ b/kerberos_helper.py @@ -0,0 +1,150 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re + +from clusterdock.utils import wait_for_condition + +KDC_IMAGE_NAME = 'clusterdock/topology_nodebase_kerberos:centos6.6' + +KERBEROS_CONFIG_CONTAINER_DIR = '/etc/clusterdock/kerberos' + +KDC_ACL_FILENAME = '/var/kerberos/krb5kdc/kadm5.acl' +KDC_CONF_FILEPATH = '/var/kerberos/krb5kdc/kdc.conf' +KDC_HOSTNAME = 'kdc' +KDC_GROUPNAME = 'kdc' +# Following is the location clusterdock places the keytab file of kerberos_principals +# so that client nodes like SDC docker can fetch it. +KDC_KEYTAB_FILEPATH = '{}/clusterdock.keytab'.format(KERBEROS_CONFIG_CONTAINER_DIR) +KRB5_CONF_FILEPATH = '/etc/krb5.conf' +MAPR_CONF_PATH = '/opt/mapr/conf' +MAPR_KEYTAB_FILENAME = 'mapr.keytab' +# Following is the location clusterdock places the mapr.keytab file so that client nodes like SDC docker can fetch it. +MAPR_KEYTAB_FILEPATH = '{}/{}'.format(KERBEROS_CONFIG_CONTAINER_DIR, MAPR_KEYTAB_FILENAME) +# Following is the location expected by MapR for the mapr.keytab file. +MAPR_CONF_KEYTAB_FILEPATH = '{}/{}'.format(MAPR_CONF_PATH, MAPR_KEYTAB_FILENAME) +MAPR_PRINCIPAL = 'mapr/my.cluster.com' +LINUX_USER_ID_START = 1000 + +logger = logging.getLogger('clusterdock.{}'.format(__name__)) + + +class Kerberos_Helper: + """Class to deal with kerberization of the cluster. + + Args: + network (:obj:`str`): Docker network to use. + """ + def __init__(self, network): + self.network = network + + @property + def mapr_principal(self): + return '{}@{}'.format(MAPR_PRINCIPAL, self.network.upper()) + + def configure_kdc(self, kdc_node, nodes, kerberos_principals, kerberos_ticket_lifetime, quiet): + logger.info('Updating KDC configurations ...') + realm = self.network.upper() + + logger.debug('Updating krb5.conf ...') + krb5_conf = kdc_node.get_file(KRB5_CONF_FILEPATH) + # Here '\g<1>' represents group matched in regex which is the original default value of ticket_lifetime. + ticket_lifetime_replacement = kerberos_ticket_lifetime if kerberos_ticket_lifetime else '\g<1>' + krb5_conf_contents = re.sub(r'EXAMPLE.COM', realm, + re.sub(r'example.com', self.network, + re.sub(r'ticket_lifetime = ((.)*)', + r'ticket_lifetime = {}'.format(ticket_lifetime_replacement), + re.sub(r'kerberos.example.com', + kdc_node.fqdn, + krb5_conf)))) + kdc_node.put_file(KRB5_CONF_FILEPATH, krb5_conf_contents) + + logger.debug('Updating kdc.conf ...') + kdc_conf = kdc_node.get_file(KDC_CONF_FILEPATH) + max_time_replacement = kerberos_ticket_lifetime if kerberos_ticket_lifetime else '1d' + kdc_node.put_file(KDC_CONF_FILEPATH, + re.sub(r'EXAMPLE.COM', realm, + re.sub(r'\[kdcdefaults\]', + r'[kdcdefaults]\n max_renewablelife = 7d\n max_life = {}'.format(max_time_replacement), + kdc_conf))) + + logger.debug('Updating kadm5.acl ...') + kadm5_acl = kdc_node.get_file(KDC_ACL_FILENAME) + kdc_node.put_file(KDC_ACL_FILENAME, + re.sub(r'EXAMPLE.COM', realm, kadm5_acl)) + + logger.info('Starting KDC ...') + + kdc_commands = [ + 'kdb5_util create -s -r {} -P kdcadmin'.format(realm), + 'kadmin.local -q "addprinc -pw {} admin/admin@{}"'.format('acladmin', realm), + 'kadmin.local -q "addprinc -randkey {}"'.format(self.mapr_principal), + 'kadmin.local -q "ktadd -k {} {}"'.format(MAPR_KEYTAB_FILEPATH, self.mapr_principal) + ] + + if kerberos_principals: + principals = ['{}@{}'.format(primary, realm) + for primary in kerberos_principals.split(',')] + if kerberos_ticket_lifetime: + kdc_commands.extend([('kadmin.local -q "addprinc -maxlife {}sec ' + '-maxrenewlife 5day -randkey {}"'.format(kerberos_ticket_lifetime, principal)) + for principal in principals]) + else: + kdc_commands.extend(['kadmin.local -q "addprinc -randkey {}"'.format(principal) + for principal in principals]) + kdc_commands.append('kadmin.local -q ' + '"xst -norandkey -k {} {}"'.format(KDC_KEYTAB_FILEPATH, + ' '.join(principals))) + kdc_commands.extend(['service krb5kdc start', + 'service kadmin start', + 'authconfig --enablekrb5 --update', + 'cp -f {} {}'.format(KRB5_CONF_FILEPATH, + KERBEROS_CONFIG_CONTAINER_DIR)]) + if kerberos_principals: + kdc_commands.append('chmod 644 {}'.format(KDC_KEYTAB_FILEPATH)) + + kdc_node.execute(' && '.join(kdc_commands), quiet=quiet) + + _validate_service_health(kdc_node, ['krb5kdc', 'kadmin'], quiet=quiet) + + # nodes are the primary and secondary nodes. + for node in nodes: + # Copy mapr.keytab file to location where MapR expects + commands = ['cp -f {} {}/{}'.format(MAPR_KEYTAB_FILEPATH, MAPR_CONF_PATH, MAPR_KEYTAB_FILENAME), + 'chown mapr:mapr {}'.format(MAPR_CONF_KEYTAB_FILEPATH)] + # Copy krb5.conf to each node's /etc/krb5.conf + commands.append('cp -f {}/{} {}'.format(KERBEROS_CONFIG_CONTAINER_DIR, 'krb5.conf', KRB5_CONF_FILEPATH)) + node.execute(' && '.join(commands), quiet=quiet) + +def _validate_service_health(kdc_node, services, quiet=True): + logger.info('Validating health of Kerberos services ...') + + def condition(node, services, quiet): + services_with_poor_health = [service + for service in services + if node.execute(command='service {} status'.format(service), + quiet=quiet).exit_code != 0] + if services_with_poor_health: + logger.debug('Services with poor health: %s', + ', '.join(services_with_poor_health)) + # Return True if the list of services with poor health is empty. + return not bool(services_with_poor_health) + wait_for_condition(condition=condition, condition_args=[kdc_node, services, quiet]) + +def create_kerberos_cluster_users(nodes, kerberos_principals, quiet): + commands = ['useradd -u {} -g hadoop {}'.format(uid, primary) + for uid, primary in enumerate(kerberos_principals.split(','), + start=LINUX_USER_ID_START)] + for node in nodes: + node.execute('; '.join(commands), quiet=quiet) diff --git a/start.py b/start.py index baa2e49..0fae1dd 100644 --- a/start.py +++ b/start.py @@ -20,19 +20,21 @@ from clusterdock.models import Cluster, Node from clusterdock.utils import wait_for_condition +from . import kerberos_helper + DEFAULT_NAMESPACE = 'clusterdock' EARLIEST_MAPR_VERSION_WITH_LICENSE_AND_CENTOS_7 = (6, 0, 0) MAPR_CONFIG_DIR = '/opt/mapr/conf' MAPR_SERVERTICKET_FILE = 'maprserverticket' MCS_SERVER_PORT = 8443 SECURE_CONFIG_CONTAINER_DIR = '/etc/clusterdock/secure' -SSL_KEYSTORE_FILE = 'ssl_keystore' -SSL_TRUSTSTORE_FILE = 'ssl_truststore' +SSL_KEYSTORE_FILES = 'ssl_keystore' +SSL_TRUSTSTORE_FILES = 'ssl_truststore' SECURE_FILES = [ MAPR_SERVERTICKET_FILE, - SSL_KEYSTORE_FILE, - SSL_TRUSTSTORE_FILE + SSL_KEYSTORE_FILES, + SSL_TRUSTSTORE_FILES ] @@ -68,7 +70,7 @@ def main(args): # Secure cluster needs the ticket to execute rest of commands # after cluster start. environment=['MAPR_TICKETFILE_LOCATION=/opt/mapr/conf/mapruserticket'] - if args.secure else []) + if args.secure or args.kerberos else []) secondary_nodes = [Node(hostname=hostname, group='secondary', @@ -76,9 +78,21 @@ def main(args): devices=node_disks.get(hostname)) for hostname in args.secondary_nodes] - cluster = Cluster(primary_node, *secondary_nodes) + nodes = [primary_node] + secondary_nodes + if args.kerberos: + logger.info('Creating KDC node...') + kerberos_helper_instance = kerberos_helper.Kerberos_Helper(args.network) + kerberos_config_host_dir = os.path.realpath(os.path.expanduser(args.clusterdock_config_directory)) + volumes = [{kerberos_config_host_dir: kerberos_helper.KERBEROS_CONFIG_CONTAINER_DIR}] + for node in nodes: + node.volumes.extend(volumes) + + kdc_node = Node(hostname=kerberos_helper.KDC_HOSTNAME, group=kerberos_helper.KDC_GROUPNAME, + image=kerberos_helper.KDC_IMAGE_NAME, volumes=volumes) + + cluster = Cluster(*nodes + ([kdc_node] if args.kerberos else [])) - if args.secure: + if args.secure or args.kerberos: secure_config_host_dir = os.path.expanduser(args.secure_config_directory) volumes = [{secure_config_host_dir: SECURE_CONFIG_CONTAINER_DIR}] for node in cluster.nodes: @@ -95,10 +109,22 @@ def main(args): cluster.primary_node = primary_node cluster.start(args.network, pull_images=args.always_pull) + # Keep track of whether to suppress DEBUG-level output in commands. + quiet = not args.verbose + + if args.kerberos: + cluster.kdc_node = kdc_node + kerberos_helper_instance.configure_kdc(kdc_node, nodes, + args.kerberos_principals, + args.kerberos_ticket_lifetime, + quiet=quiet) + if args.kerberos_principals: + kerberos_helper.create_kerberos_cluster_users(nodes, args.kerberos_principals, quiet=quiet) + logger.info('Generating new UUIDs ...') cluster.execute('/opt/mapr/server/mruuidgen > /opt/mapr/hostid') - if not args.secure: + if not (args.secure or args.kerberos): logger.info('Configuring the cluster ...') for node in cluster: configure_command = ('/opt/mapr/server/configure.sh -C {0} -Z {0} -RM {0} -HS {0} ' @@ -116,7 +142,7 @@ def main(args): )) source_files = ['{}/{}'.format(MAPR_CONFIG_DIR, file) for file in SECURE_FILES] commands = [configure_command, - 'chmod 600 {}/{}'.format(MAPR_CONFIG_DIR, SSL_KEYSTORE_FILE), + 'chmod 600 {}/{}'.format(MAPR_CONFIG_DIR, SSL_KEYSTORE_FILES), 'cp -f {src} {dest_dir}'.format(src=' '.join(source_files), dest_dir=SECURE_CONFIG_CONTAINER_DIR)] primary_node.execute(' && '.join(commands)) @@ -149,6 +175,15 @@ def failure(timeout): time_between_checks=3, timeout=180, success=success, failure=failure) mcs_server_host_port = primary_node.host_ports.get(MCS_SERVER_PORT) + _configure_after_mcs_server_start(primary_node, secondary_nodes, args, mapr_version_tuple, + kerberos_helper_instance.mapr_principal if args.kerberos else None) + + logger.info('MapR Control System server is now accessible at https://%s:%s', + getfqdn(), mcs_server_host_port) + + +def _configure_after_mcs_server_start(primary_node, secondary_nodes, args, + mapr_version_tuple, mapr_principal=None): logger.info('Creating /apps/spark directory on %s ...', primary_node.hostname) spark_directory_command = ['hadoop fs -mkdir -p /apps/spark', 'hadoop fs -chmod 777 /apps/spark'] @@ -159,21 +194,59 @@ def failure(timeout): '-produceperm p -consumeperm p -topicperm p') if mapr_version_tuple >= EARLIEST_MAPR_VERSION_WITH_LICENSE_AND_CENTOS_7 and args.license_url: - license_commands = ['curl --user {} {} > /tmp/lic'.format(args.license_credentials, - args.license_url), - '/opt/mapr/bin/maprcli license add -license /tmp/lic -is_file true', - 'rm -rf /tmp/lic'] - logger.info('Applying license ...') - primary_node.execute(' && '.join(license_commands)) + _apply_license(args, primary_node) if not args.dont_register_gateway: - logger.info('Registering gateway with the cluster ...') - register_gateway_commands = ["cat /opt/mapr/conf/mapr-clusters.conf | egrep -o '^[^ ]* '" - ' > /tmp/cluster-name', - 'maprcli cluster gateway set -dstcluster $(cat ' - '/tmp/cluster-name) -gateways {}'.format(primary_node.fqdn), - 'rm /tmp/cluster-name'] - primary_node.execute(' && '.join(register_gateway_commands)) + _register_gateway(primary_node) - logger.info('MapR Control System server is now accessible at https://%s:%s', - getfqdn(), mcs_server_host_port) + if args.secure or args.kerberos: + primary_node.execute('echo mapr | sudo -u mapr maprlogin password') + + if args.kerberos: + _kerberize_cluster(primary_node, secondary_nodes, mapr_principal) + + +def _kerberize_cluster(primary_node, secondary_nodes, mapr_principal): + commands = ['service mapr-warden stop', + 'service mapr-zookeeper stop', + ('/opt/mapr/server/configure.sh -K -P {0} -C {1} -Z {1} ' + .format(mapr_principal, primary_node.fqdn))] + primary_node.execute(' && '.join(commands)) + for node in secondary_nodes: + node.execute(commands[2]) + + logger.info('After kerberization, waiting for MapR Control System server to come online ...') + + def condition(address, port): + return socket().connect_ex((address, port)) == 0 + + def success(time): + logger.info('MapR Control System server is online after %s seconds.', time) + + def failure(timeout): + raise TimeoutError('Timed out after {} seconds waiting ' + 'for MapR Control System server to come online.'.format(timeout)) + wait_for_condition(condition=condition, + condition_args=[primary_node.ip_address, MCS_SERVER_PORT], + time_between_checks=3, timeout=180, success=success, failure=failure) + primary_node.execute('kinit -kt {} {}'.format(kerberos_helper.MAPR_CONF_KEYTAB_FILEPATH, mapr_principal)) + primary_node.execute('maprlogin kerberos') + + +def _apply_license(args, primary_node): + license_commands = ['curl --user {} {} > /tmp/lic'.format(args.license_credentials, + args.license_url), + '/opt/mapr/bin/maprcli license add -license /tmp/lic -is_file true', + 'rm -rf /tmp/lic'] + logger.info('Applying license ...') + primary_node.execute(' && '.join(license_commands)) + + +def _register_gateway(primary_node): + logger.info('Registering gateway with the cluster ...') + register_gateway_commands = ["cat /opt/mapr/conf/mapr-clusters.conf | egrep -o '^[^ ]* '" + ' > /tmp/cluster-name', + 'maprcli cluster gateway set -dstcluster $(cat ' + '/tmp/cluster-name) -gateways {}'.format(primary_node.fqdn), + 'rm /tmp/cluster-name'] + primary_node.execute(' && '.join(register_gateway_commands)) diff --git a/topology.yaml b/topology.yaml index a0d3534..afe4929 100644 --- a/topology.yaml +++ b/topology.yaml @@ -27,6 +27,16 @@ build args: metavar: ver start args: + --kerberos, -k: + action: store_true + help: If specified, enable Kerberos for the cluster + --kerberos-principals: + help: If specified, a comma-separated list of Kerberos user principals to create in KDC + metavar: princ1,princ2,... + --kerberos-ticket-lifetime: + default: 86400 + help: If specified, the maximum time period in seconds for which a ticket may be valid in the realm + metavar: seconds --license-credentials: help: MapR license credentials to use in the format username:password metavar: credentials