Current File : //lib/python2.7/site-packages/salt/pillar/file_tree.py
# -*- coding: utf-8 -*-
'''
The ``file_tree`` external pillar allows values from all files in a directory
tree to be imported as Pillar data.
.. note::
This is an external pillar and is subject to the :ref:`rules and
constraints <external-pillars>` governing external pillars.
.. versionadded:: 2015.5.0
In this pillar, data is organized by either Minion ID or Nodegroup name. To
setup pillar data for a specific Minion, place it in
``<root_dir>/hosts/<minion_id>``. To setup pillar data for an entire
Nodegroup, place it in ``<root_dir>/nodegroups/<node_group>`` where
``<node_group>`` is the Nodegroup's name.
Example ``file_tree`` Pillar
============================
Master Configuration
--------------------
.. code-block:: yaml
ext_pillar:
- file_tree:
root_dir: /srv/ext_pillar
follow_dir_links: False
keep_newline: True
The ``root_dir`` parameter is required and points to the directory where files
for each host are stored. The ``follow_dir_links`` parameter is optional and
defaults to False. If ``follow_dir_links`` is set to True, this external pillar
will follow symbolic links to other directories.
.. warning::
Be careful when using ``follow_dir_links``, as a recursive symlink chain
will result in unexpected results.
.. versionchanged:: 2018.3.0
If ``root_dir`` is a relative path, it will be treated as relative to the
:conf_master:`pillar_roots` of the environment specified by
:conf_minion:`pillarenv`. If an environment specifies multiple
roots, this module will search for files relative to all of them, in order,
merging the results.
If ``keep_newline`` is set to ``True``, then the pillar values for files ending
in newlines will keep that newline. The default behavior is to remove the
end-of-file newline. ``keep_newline`` should be turned on if the pillar data is
intended to be used to deploy a file using ``contents_pillar`` with a
:py:func:`file.managed <salt.states.file.managed>` state.
.. versionchanged:: 2015.8.4
The ``raw_data`` parameter has been renamed to ``keep_newline``. In earlier
releases, ``raw_data`` must be used. Also, this parameter can now be a list
of globs, allowing for more granular control over which pillar values keep
their end-of-file newline. The globs match paths relative to the
directories named for minion IDs and nodegroups underneath the ``root_dir``
(see the layout examples in the below sections).
.. code-block:: yaml
ext_pillar:
- file_tree:
root_dir: /path/to/root/directory
keep_newline:
- files/testdir/*
.. note::
In earlier releases, this documentation incorrectly stated that binary
files would not affected by the ``keep_newline`` configuration. However,
this module does not actually distinguish between binary and text files.
.. versionchanged:: 2017.7.0
Templating/rendering has been added. You can now specify a default render
pipeline and a black- and whitelist of (dis)allowed renderers.
``template`` must be set to ``True`` for templating to happen.
.. code-block:: yaml
ext_pillar:
- file_tree:
root_dir: /path/to/root/directory
render_default: jinja|yaml
renderer_blacklist:
- gpg
renderer_whitelist:
- jinja
- yaml
template: True
Assigning Pillar Data to Individual Hosts
-----------------------------------------
To configure pillar data for each host, this external pillar will recursively
iterate over ``root_dir``/hosts/``id`` (where ``id`` is a minion ID), and
compile pillar data with each subdirectory as a dictionary key and each file
as a value.
For example, the following ``root_dir`` tree:
.. code-block:: text
./hosts/
./hosts/test-host/
./hosts/test-host/files/
./hosts/test-host/files/testdir/
./hosts/test-host/files/testdir/file1.txt
./hosts/test-host/files/testdir/file2.txt
./hosts/test-host/files/another-testdir/
./hosts/test-host/files/another-testdir/symlink-to-file1.txt
will result in the following pillar tree for minion with ID ``test-host``:
.. code-block:: text
test-host:
----------
apache:
----------
config.d:
----------
00_important.conf:
<important_config important_setting="yes" />
20_bob_extra.conf:
<bob_specific_cfg has_freeze_ray="yes" />
corporate_app:
----------
settings:
----------
common_settings:
// This is the main settings file for the corporate
// internal web app
main_setting: probably
bob_settings:
role: bob
.. note::
The leaf data in the example shown is the contents of the pillar files.
'''
from __future__ import absolute_import, print_function, unicode_literals
# Import python libs
import fnmatch
import logging
import os
# Import salt libs
import salt.loader
import salt.utils.dictupdate
import salt.utils.files
import salt.utils.minions
import salt.utils.path
import salt.utils.stringio
import salt.utils.stringutils
import salt.template
from salt.ext import six
# Set up logging
log = logging.getLogger(__name__)
def _on_walk_error(err):
'''
Log salt.utils.path.os_walk() error.
'''
log.error('%s: %s', err.filename, err.strerror)
def _check_newline(prefix, file_name, keep_newline):
'''
Return a boolean stating whether or not a file's trailing newline should be
removed. To figure this out, first check if keep_newline is a boolean and
if so, return its opposite. Otherwise, iterate over keep_newline and check
if any of the patterns match the file path. If a match is found, return
False, otherwise return True.
'''
if isinstance(keep_newline, bool):
return not keep_newline
full_path = os.path.join(prefix, file_name)
for pattern in keep_newline:
try:
if fnmatch.fnmatch(full_path, pattern):
return False
except TypeError:
if fnmatch.fnmatch(full_path, six.text_type(pattern)):
return False
return True
def _construct_pillar(top_dir,
follow_dir_links,
keep_newline=False,
render_default=None,
renderer_blacklist=None,
renderer_whitelist=None,
template=False):
'''
Construct pillar from file tree.
'''
pillar = {}
renderers = salt.loader.render(__opts__, __salt__)
norm_top_dir = os.path.normpath(top_dir)
for dir_path, dir_names, file_names in salt.utils.path.os_walk(
top_dir, topdown=True, onerror=_on_walk_error,
followlinks=follow_dir_links):
# Find current path in pillar tree
pillar_node = pillar
norm_dir_path = os.path.normpath(dir_path)
prefix = os.path.relpath(norm_dir_path, norm_top_dir)
if norm_dir_path != norm_top_dir:
path_parts = []
head = prefix
while head:
head, tail = os.path.split(head)
path_parts.insert(0, tail)
while path_parts:
pillar_node = pillar_node[path_parts.pop(0)]
# Create dicts for subdirectories
for dir_name in dir_names:
pillar_node[dir_name] = {}
# Add files
for file_name in file_names:
file_path = os.path.join(dir_path, file_name)
if not os.path.isfile(file_path):
log.error('file_tree: %s: not a regular file', file_path)
continue
contents = b''
try:
with salt.utils.files.fopen(file_path, 'rb') as fhr:
buf = fhr.read(__opts__['file_buffer_size'])
while buf:
contents += buf
buf = fhr.read(__opts__['file_buffer_size'])
if contents.endswith(b'\n') \
and _check_newline(prefix,
file_name,
keep_newline):
contents = contents[:-1]
except (IOError, OSError) as exc:
log.error('file_tree: Error reading %s: %s',
file_path,
exc.strerror)
else:
data = contents
if template is True:
data = salt.template.compile_template_str(template=salt.utils.stringutils.to_unicode(contents),
renderers=renderers,
default=render_default,
blacklist=renderer_blacklist,
whitelist=renderer_whitelist)
if salt.utils.stringio.is_readable(data):
pillar_node[file_name] = data.getvalue()
else:
pillar_node[file_name] = data
return pillar
def ext_pillar(minion_id,
pillar,
root_dir=None,
follow_dir_links=False,
debug=False,
keep_newline=False,
render_default=None,
renderer_blacklist=None,
renderer_whitelist=None,
template=False):
'''
Compile pillar data from the given ``root_dir`` specific to Nodegroup names
and Minion IDs.
If a Minion's ID is not found at ``<root_dir>/host/<minion_id>`` or if it
is not included in any Nodegroups named at
``<root_dir>/nodegroups/<node_group>``, no pillar data provided by this
pillar module will be available for that Minion.
.. versionchanged:: 2017.7.0
Templating/rendering has been added. You can now specify a default
render pipeline and a black- and whitelist of (dis)allowed renderers.
``template`` must be set to ``True`` for templating to happen.
.. code-block:: yaml
ext_pillar:
- file_tree:
root_dir: /path/to/root/directory
render_default: jinja|yaml
renderer_blacklist:
- gpg
renderer_whitelist:
- jinja
- yaml
template: True
:param minion_id:
The ID of the Minion whose pillar data is to be collected
:param pillar:
Unused by the ``file_tree`` pillar module
:param root_dir:
Filesystem directory used as the root for pillar data (e.g.
``/srv/ext_pillar``)
.. versionchanged:: 2018.3.0
If ``root_dir`` is a relative path, it will be treated as relative to the
:conf_master:`pillar_roots` of the environment specified by
:conf_minion:`pillarenv`. If an environment specifies multiple
roots, this module will search for files relative to all of them, in order,
merging the results.
:param follow_dir_links:
Follow symbolic links to directories while collecting pillar files.
Defaults to ``False``.
.. warning::
Care should be exercised when enabling this option as it will
follow links that point outside of ``root_dir``.
.. warning::
Symbolic links that lead to infinite recursion are not filtered.
:param debug:
Enable debug information at log level ``debug``. Defaults to
``False``. This option may be useful to help debug errors when setting
up the ``file_tree`` pillar module.
:param keep_newline:
Preserve the end-of-file newline in files. Defaults to ``False``.
This option may either be a boolean or a list of file globs (as defined
by the `Python fnmatch package
<https://docs.python.org/library/fnmatch.html>`_) for which end-of-file
newlines are to be kept.
``keep_newline`` should be turned on if the pillar data is intended to
be used to deploy a file using ``contents_pillar`` with a
:py:func:`file.managed <salt.states.file.managed>` state.
.. versionchanged:: 2015.8.4
The ``raw_data`` parameter has been renamed to ``keep_newline``. In
earlier releases, ``raw_data`` must be used. Also, this parameter
can now be a list of globs, allowing for more granular control over
which pillar values keep their end-of-file newline. The globs match
paths relative to the directories named for Minion IDs and
Nodegroup namess underneath the ``root_dir``.
.. code-block:: yaml
ext_pillar:
- file_tree:
root_dir: /srv/ext_pillar
keep_newline:
- apache/config.d/*
- corporate_app/settings/*
.. note::
In earlier releases, this documentation incorrectly stated that
binary files would not affected by the ``keep_newline``. However,
this module does not actually distinguish between binary and text
files.
:param render_default:
Override Salt's :conf_master:`default global renderer <renderer>` for
the ``file_tree`` pillar.
.. code-block:: yaml
render_default: jinja
:param renderer_blacklist:
Disallow renderers for pillar files.
.. code-block:: yaml
renderer_blacklist:
- json
:param renderer_whitelist:
Allow renderers for pillar files.
.. code-block:: yaml
renderer_whitelist:
- yaml
- jinja
:param template:
Enable templating of pillar files. Defaults to ``False``.
'''
# Not used
del pillar
if not root_dir:
log.error('file_tree: no root_dir specified')
return {}
if not os.path.isabs(root_dir):
pillarenv = __opts__['pillarenv']
if pillarenv is None:
log.error('file_tree: root_dir is relative but pillarenv is not set')
return {}
log.debug('file_tree: pillarenv = %s', pillarenv)
env_roots = __opts__['pillar_roots'].get(pillarenv, None)
if env_roots is None:
log.error('file_tree: root_dir is relative but no pillar_roots are specified '
' for pillarenv %s', pillarenv)
return {}
env_dirs = []
for env_root in env_roots:
env_dir = os.path.normpath(os.path.join(env_root, root_dir))
# don't redundantly load consecutively, but preserve any expected precedence
if env_dir not in env_dirs or env_dir != env_dirs[-1]:
env_dirs.append(env_dir)
dirs = env_dirs
else:
dirs = [root_dir]
result_pillar = {}
for root in dirs:
dir_pillar = _ext_pillar(minion_id,
root,
follow_dir_links,
debug,
keep_newline,
render_default,
renderer_blacklist,
renderer_whitelist,
template)
result_pillar = salt.utils.dictupdate.merge(result_pillar,
dir_pillar,
strategy='recurse')
return result_pillar
def _ext_pillar(minion_id,
root_dir,
follow_dir_links,
debug,
keep_newline,
render_default,
renderer_blacklist,
renderer_whitelist,
template):
'''
Compile pillar data for a single root_dir for the specified minion ID
'''
log.debug('file_tree: reading %s', root_dir)
if not os.path.isdir(root_dir):
log.error(
'file_tree: root_dir %s does not exist or is not a directory',
root_dir
)
return {}
if not isinstance(keep_newline, (bool, list)):
log.error(
'file_tree: keep_newline must be either True/False or a list '
'of file globs. Skipping this ext_pillar for root_dir %s',
root_dir
)
return {}
ngroup_pillar = {}
nodegroups_dir = os.path.join(root_dir, 'nodegroups')
if os.path.exists(nodegroups_dir) and len(__opts__.get('nodegroups', ())) > 0:
master_ngroups = __opts__['nodegroups']
ext_pillar_dirs = os.listdir(nodegroups_dir)
if len(ext_pillar_dirs) > 0:
for nodegroup in ext_pillar_dirs:
if (os.path.isdir(nodegroups_dir) and
nodegroup in master_ngroups):
ckminions = salt.utils.minions.CkMinions(__opts__)
_res = ckminions.check_minions(
master_ngroups[nodegroup],
'compound')
match = _res['minions']
if minion_id in match:
ngroup_dir = os.path.join(
nodegroups_dir, six.text_type(nodegroup))
ngroup_pillar = salt.utils.dictupdate.merge(ngroup_pillar,
_construct_pillar(ngroup_dir,
follow_dir_links,
keep_newline,
render_default,
renderer_blacklist,
renderer_whitelist,
template),
strategy='recurse'
)
else:
if debug is True:
log.debug(
'file_tree: no nodegroups found in file tree directory %s, skipping...',
ext_pillar_dirs
)
else:
if debug is True:
log.debug('file_tree: no nodegroups found in master configuration')
host_dir = os.path.join(root_dir, 'hosts', minion_id)
if not os.path.exists(host_dir):
if debug is True:
log.debug(
'file_tree: no pillar data for minion %s found in file tree directory %s',
minion_id,
host_dir
)
return ngroup_pillar
if not os.path.isdir(host_dir):
log.error('file_tree: %s exists, but is not a directory', host_dir)
return ngroup_pillar
host_pillar = _construct_pillar(host_dir,
follow_dir_links,
keep_newline,
render_default,
renderer_blacklist,
renderer_whitelist,
template)
return salt.utils.dictupdate.merge(ngroup_pillar,
host_pillar,
strategy='recurse')
Mini Shell