394 lines
14 KiB
Python
394 lines
14 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
#
|
||
|
|
# Copyright (C) 2012-2013 Python Software Foundation.
|
||
|
|
# See LICENSE.txt and CONTRIBUTORS.txt.
|
||
|
|
#
|
||
|
|
"""
|
||
|
|
Class representing the list of files in a distribution.
|
||
|
|
|
||
|
|
Equivalent to distutils.filelist, but fixes some problems.
|
||
|
|
"""
|
||
|
|
import fnmatch
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
|
||
|
|
from . import DistlibException
|
||
|
|
from .compat import fsdecode
|
||
|
|
from .util import convert_path
|
||
|
|
|
||
|
|
|
||
|
|
__all__ = ['Manifest']
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
# a \ followed by some spaces + EOL
|
||
|
|
_COLLAPSE_PATTERN = re.compile('\\\\w*\n', re.M)
|
||
|
|
_COMMENTED_LINE = re.compile('#.*?(?=\n)|\n(?=$)', re.M | re.S)
|
||
|
|
|
||
|
|
#
|
||
|
|
# Due to the different results returned by fnmatch.translate, we need
|
||
|
|
# to do slightly different processing for Python 2.7 and 3.2 ... this needed
|
||
|
|
# to be brought in for Python 3.6 onwards.
|
||
|
|
#
|
||
|
|
_PYTHON_VERSION = sys.version_info[:2]
|
||
|
|
|
||
|
|
class Manifest(object):
|
||
|
|
"""A list of files built by on exploring the filesystem and filtered by
|
||
|
|
applying various patterns to what we find there.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, base=None):
|
||
|
|
"""
|
||
|
|
Initialise an instance.
|
||
|
|
|
||
|
|
:param base: The base directory to explore under.
|
||
|
|
"""
|
||
|
|
self.base = os.path.abspath(os.path.normpath(base or os.getcwd()))
|
||
|
|
self.prefix = self.base + os.sep
|
||
|
|
self.allfiles = None
|
||
|
|
self.files = set()
|
||
|
|
|
||
|
|
#
|
||
|
|
# Public API
|
||
|
|
#
|
||
|
|
|
||
|
|
def findall(self):
|
||
|
|
"""Find all files under the base and set ``allfiles`` to the absolute
|
||
|
|
pathnames of files found.
|
||
|
|
"""
|
||
|
|
from stat import S_ISREG, S_ISDIR, S_ISLNK
|
||
|
|
|
||
|
|
self.allfiles = allfiles = []
|
||
|
|
root = self.base
|
||
|
|
stack = [root]
|
||
|
|
pop = stack.pop
|
||
|
|
push = stack.append
|
||
|
|
|
||
|
|
while stack:
|
||
|
|
root = pop()
|
||
|
|
names = os.listdir(root)
|
||
|
|
|
||
|
|
for name in names:
|
||
|
|
fullname = os.path.join(root, name)
|
||
|
|
|
||
|
|
# Avoid excess stat calls -- just one will do, thank you!
|
||
|
|
stat = os.stat(fullname)
|
||
|
|
mode = stat.st_mode
|
||
|
|
if S_ISREG(mode):
|
||
|
|
allfiles.append(fsdecode(fullname))
|
||
|
|
elif S_ISDIR(mode) and not S_ISLNK(mode):
|
||
|
|
push(fullname)
|
||
|
|
|
||
|
|
def add(self, item):
|
||
|
|
"""
|
||
|
|
Add a file to the manifest.
|
||
|
|
|
||
|
|
:param item: The pathname to add. This can be relative to the base.
|
||
|
|
"""
|
||
|
|
if not item.startswith(self.prefix):
|
||
|
|
item = os.path.join(self.base, item)
|
||
|
|
self.files.add(os.path.normpath(item))
|
||
|
|
|
||
|
|
def add_many(self, items):
|
||
|
|
"""
|
||
|
|
Add a list of files to the manifest.
|
||
|
|
|
||
|
|
:param items: The pathnames to add. These can be relative to the base.
|
||
|
|
"""
|
||
|
|
for item in items:
|
||
|
|
self.add(item)
|
||
|
|
|
||
|
|
def sorted(self, wantdirs=False):
|
||
|
|
"""
|
||
|
|
Return sorted files in directory order
|
||
|
|
"""
|
||
|
|
|
||
|
|
def add_dir(dirs, d):
|
||
|
|
dirs.add(d)
|
||
|
|
logger.debug('add_dir added %s', d)
|
||
|
|
if d != self.base:
|
||
|
|
parent, _ = os.path.split(d)
|
||
|
|
assert parent not in ('', '/')
|
||
|
|
add_dir(dirs, parent)
|
||
|
|
|
||
|
|
result = set(self.files) # make a copy!
|
||
|
|
if wantdirs:
|
||
|
|
dirs = set()
|
||
|
|
for f in result:
|
||
|
|
add_dir(dirs, os.path.dirname(f))
|
||
|
|
result |= dirs
|
||
|
|
return [os.path.join(*path_tuple) for path_tuple in
|
||
|
|
sorted(os.path.split(path) for path in result)]
|
||
|
|
|
||
|
|
def clear(self):
|
||
|
|
"""Clear all collected files."""
|
||
|
|
self.files = set()
|
||
|
|
self.allfiles = []
|
||
|
|
|
||
|
|
def process_directive(self, directive):
|
||
|
|
"""
|
||
|
|
Process a directive which either adds some files from ``allfiles`` to
|
||
|
|
``files``, or removes some files from ``files``.
|
||
|
|
|
||
|
|
:param directive: The directive to process. This should be in a format
|
||
|
|
compatible with distutils ``MANIFEST.in`` files:
|
||
|
|
|
||
|
|
http://docs.python.org/distutils/sourcedist.html#commands
|
||
|
|
"""
|
||
|
|
# Parse the line: split it up, make sure the right number of words
|
||
|
|
# is there, and return the relevant words. 'action' is always
|
||
|
|
# defined: it's the first word of the line. Which of the other
|
||
|
|
# three are defined depends on the action; it'll be either
|
||
|
|
# patterns, (dir and patterns), or (dirpattern).
|
||
|
|
action, patterns, thedir, dirpattern = self._parse_directive(directive)
|
||
|
|
|
||
|
|
# OK, now we know that the action is valid and we have the
|
||
|
|
# right number of words on the line for that action -- so we
|
||
|
|
# can proceed with minimal error-checking.
|
||
|
|
if action == 'include':
|
||
|
|
for pattern in patterns:
|
||
|
|
if not self._include_pattern(pattern, anchor=True):
|
||
|
|
logger.warning('no files found matching %r', pattern)
|
||
|
|
|
||
|
|
elif action == 'exclude':
|
||
|
|
for pattern in patterns:
|
||
|
|
found = self._exclude_pattern(pattern, anchor=True)
|
||
|
|
#if not found:
|
||
|
|
# logger.warning('no previously-included files '
|
||
|
|
# 'found matching %r', pattern)
|
||
|
|
|
||
|
|
elif action == 'global-include':
|
||
|
|
for pattern in patterns:
|
||
|
|
if not self._include_pattern(pattern, anchor=False):
|
||
|
|
logger.warning('no files found matching %r '
|
||
|
|
'anywhere in distribution', pattern)
|
||
|
|
|
||
|
|
elif action == 'global-exclude':
|
||
|
|
for pattern in patterns:
|
||
|
|
found = self._exclude_pattern(pattern, anchor=False)
|
||
|
|
#if not found:
|
||
|
|
# logger.warning('no previously-included files '
|
||
|
|
# 'matching %r found anywhere in '
|
||
|
|
# 'distribution', pattern)
|
||
|
|
|
||
|
|
elif action == 'recursive-include':
|
||
|
|
for pattern in patterns:
|
||
|
|
if not self._include_pattern(pattern, prefix=thedir):
|
||
|
|
logger.warning('no files found matching %r '
|
||
|
|
'under directory %r', pattern, thedir)
|
||
|
|
|
||
|
|
elif action == 'recursive-exclude':
|
||
|
|
for pattern in patterns:
|
||
|
|
found = self._exclude_pattern(pattern, prefix=thedir)
|
||
|
|
#if not found:
|
||
|
|
# logger.warning('no previously-included files '
|
||
|
|
# 'matching %r found under directory %r',
|
||
|
|
# pattern, thedir)
|
||
|
|
|
||
|
|
elif action == 'graft':
|
||
|
|
if not self._include_pattern(None, prefix=dirpattern):
|
||
|
|
logger.warning('no directories found matching %r',
|
||
|
|
dirpattern)
|
||
|
|
|
||
|
|
elif action == 'prune':
|
||
|
|
if not self._exclude_pattern(None, prefix=dirpattern):
|
||
|
|
logger.warning('no previously-included directories found '
|
||
|
|
'matching %r', dirpattern)
|
||
|
|
else: # pragma: no cover
|
||
|
|
# This should never happen, as it should be caught in
|
||
|
|
# _parse_template_line
|
||
|
|
raise DistlibException(
|
||
|
|
'invalid action %r' % action)
|
||
|
|
|
||
|
|
#
|
||
|
|
# Private API
|
||
|
|
#
|
||
|
|
|
||
|
|
def _parse_directive(self, directive):
|
||
|
|
"""
|
||
|
|
Validate a directive.
|
||
|
|
:param directive: The directive to validate.
|
||
|
|
:return: A tuple of action, patterns, thedir, dir_patterns
|
||
|
|
"""
|
||
|
|
words = directive.split()
|
||
|
|
if len(words) == 1 and words[0] not in ('include', 'exclude',
|
||
|
|
'global-include',
|
||
|
|
'global-exclude',
|
||
|
|
'recursive-include',
|
||
|
|
'recursive-exclude',
|
||
|
|
'graft', 'prune'):
|
||
|
|
# no action given, let's use the default 'include'
|
||
|
|
words.insert(0, 'include')
|
||
|
|
|
||
|
|
action = words[0]
|
||
|
|
patterns = thedir = dir_pattern = None
|
||
|
|
|
||
|
|
if action in ('include', 'exclude',
|
||
|
|
'global-include', 'global-exclude'):
|
||
|
|
if len(words) < 2:
|
||
|
|
raise DistlibException(
|
||
|
|
'%r expects <pattern1> <pattern2> ...' % action)
|
||
|
|
|
||
|
|
patterns = [convert_path(word) for word in words[1:]]
|
||
|
|
|
||
|
|
elif action in ('recursive-include', 'recursive-exclude'):
|
||
|
|
if len(words) < 3:
|
||
|
|
raise DistlibException(
|
||
|
|
'%r expects <dir> <pattern1> <pattern2> ...' % action)
|
||
|
|
|
||
|
|
thedir = convert_path(words[1])
|
||
|
|
patterns = [convert_path(word) for word in words[2:]]
|
||
|
|
|
||
|
|
elif action in ('graft', 'prune'):
|
||
|
|
if len(words) != 2:
|
||
|
|
raise DistlibException(
|
||
|
|
'%r expects a single <dir_pattern>' % action)
|
||
|
|
|
||
|
|
dir_pattern = convert_path(words[1])
|
||
|
|
|
||
|
|
else:
|
||
|
|
raise DistlibException('unknown action %r' % action)
|
||
|
|
|
||
|
|
return action, patterns, thedir, dir_pattern
|
||
|
|
|
||
|
|
def _include_pattern(self, pattern, anchor=True, prefix=None,
|
||
|
|
is_regex=False):
|
||
|
|
"""Select strings (presumably filenames) from 'self.files' that
|
||
|
|
match 'pattern', a Unix-style wildcard (glob) pattern.
|
||
|
|
|
||
|
|
Patterns are not quite the same as implemented by the 'fnmatch'
|
||
|
|
module: '*' and '?' match non-special characters, where "special"
|
||
|
|
is platform-dependent: slash on Unix; colon, slash, and backslash on
|
||
|
|
DOS/Windows; and colon on Mac OS.
|
||
|
|
|
||
|
|
If 'anchor' is true (the default), then the pattern match is more
|
||
|
|
stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
|
||
|
|
'anchor' is false, both of these will match.
|
||
|
|
|
||
|
|
If 'prefix' is supplied, then only filenames starting with 'prefix'
|
||
|
|
(itself a pattern) and ending with 'pattern', with anything in between
|
||
|
|
them, will match. 'anchor' is ignored in this case.
|
||
|
|
|
||
|
|
If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
|
||
|
|
'pattern' is assumed to be either a string containing a regex or a
|
||
|
|
regex object -- no translation is done, the regex is just compiled
|
||
|
|
and used as-is.
|
||
|
|
|
||
|
|
Selected strings will be added to self.files.
|
||
|
|
|
||
|
|
Return True if files are found.
|
||
|
|
"""
|
||
|
|
# XXX docstring lying about what the special chars are?
|
||
|
|
found = False
|
||
|
|
pattern_re = self._translate_pattern(pattern, anchor, prefix, is_regex)
|
||
|
|
|
||
|
|
# delayed loading of allfiles list
|
||
|
|
if self.allfiles is None:
|
||
|
|
self.findall()
|
||
|
|
|
||
|
|
for name in self.allfiles:
|
||
|
|
if pattern_re.search(name):
|
||
|
|
self.files.add(name)
|
||
|
|
found = True
|
||
|
|
return found
|
||
|
|
|
||
|
|
def _exclude_pattern(self, pattern, anchor=True, prefix=None,
|
||
|
|
is_regex=False):
|
||
|
|
"""Remove strings (presumably filenames) from 'files' that match
|
||
|
|
'pattern'.
|
||
|
|
|
||
|
|
Other parameters are the same as for 'include_pattern()', above.
|
||
|
|
The list 'self.files' is modified in place. Return True if files are
|
||
|
|
found.
|
||
|
|
|
||
|
|
This API is public to allow e.g. exclusion of SCM subdirs, e.g. when
|
||
|
|
packaging source distributions
|
||
|
|
"""
|
||
|
|
found = False
|
||
|
|
pattern_re = self._translate_pattern(pattern, anchor, prefix, is_regex)
|
||
|
|
for f in list(self.files):
|
||
|
|
if pattern_re.search(f):
|
||
|
|
self.files.remove(f)
|
||
|
|
found = True
|
||
|
|
return found
|
||
|
|
|
||
|
|
def _translate_pattern(self, pattern, anchor=True, prefix=None,
|
||
|
|
is_regex=False):
|
||
|
|
"""Translate a shell-like wildcard pattern to a compiled regular
|
||
|
|
expression.
|
||
|
|
|
||
|
|
Return the compiled regex. If 'is_regex' true,
|
||
|
|
then 'pattern' is directly compiled to a regex (if it's a string)
|
||
|
|
or just returned as-is (assumes it's a regex object).
|
||
|
|
"""
|
||
|
|
if is_regex:
|
||
|
|
if isinstance(pattern, str):
|
||
|
|
return re.compile(pattern)
|
||
|
|
else:
|
||
|
|
return pattern
|
||
|
|
|
||
|
|
if _PYTHON_VERSION > (3, 2):
|
||
|
|
# ditch start and end characters
|
||
|
|
start, _, end = self._glob_to_re('_').partition('_')
|
||
|
|
|
||
|
|
if pattern:
|
||
|
|
pattern_re = self._glob_to_re(pattern)
|
||
|
|
if _PYTHON_VERSION > (3, 2):
|
||
|
|
assert pattern_re.startswith(start) and pattern_re.endswith(end)
|
||
|
|
else:
|
||
|
|
pattern_re = ''
|
||
|
|
|
||
|
|
base = re.escape(os.path.join(self.base, ''))
|
||
|
|
if prefix is not None:
|
||
|
|
# ditch end of pattern character
|
||
|
|
if _PYTHON_VERSION <= (3, 2):
|
||
|
|
empty_pattern = self._glob_to_re('')
|
||
|
|
prefix_re = self._glob_to_re(prefix)[:-len(empty_pattern)]
|
||
|
|
else:
|
||
|
|
prefix_re = self._glob_to_re(prefix)
|
||
|
|
assert prefix_re.startswith(start) and prefix_re.endswith(end)
|
||
|
|
prefix_re = prefix_re[len(start): len(prefix_re) - len(end)]
|
||
|
|
sep = os.sep
|
||
|
|
if os.sep == '\\':
|
||
|
|
sep = r'\\'
|
||
|
|
if _PYTHON_VERSION <= (3, 2):
|
||
|
|
pattern_re = '^' + base + sep.join((prefix_re,
|
||
|
|
'.*' + pattern_re))
|
||
|
|
else:
|
||
|
|
pattern_re = pattern_re[len(start): len(pattern_re) - len(end)]
|
||
|
|
pattern_re = r'%s%s%s%s.*%s%s' % (start, base, prefix_re, sep,
|
||
|
|
pattern_re, end)
|
||
|
|
else: # no prefix -- respect anchor flag
|
||
|
|
if anchor:
|
||
|
|
if _PYTHON_VERSION <= (3, 2):
|
||
|
|
pattern_re = '^' + base + pattern_re
|
||
|
|
else:
|
||
|
|
pattern_re = r'%s%s%s' % (start, base, pattern_re[len(start):])
|
||
|
|
|
||
|
|
return re.compile(pattern_re)
|
||
|
|
|
||
|
|
def _glob_to_re(self, pattern):
|
||
|
|
"""Translate a shell-like glob pattern to a regular expression.
|
||
|
|
|
||
|
|
Return a string containing the regex. Differs from
|
||
|
|
'fnmatch.translate()' in that '*' does not match "special characters"
|
||
|
|
(which are platform-specific).
|
||
|
|
"""
|
||
|
|
pattern_re = fnmatch.translate(pattern)
|
||
|
|
|
||
|
|
# '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
|
||
|
|
# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
|
||
|
|
# and by extension they shouldn't match such "special characters" under
|
||
|
|
# any OS. So change all non-escaped dots in the RE to match any
|
||
|
|
# character except the special characters (currently: just os.sep).
|
||
|
|
sep = os.sep
|
||
|
|
if os.sep == '\\':
|
||
|
|
# we're using a regex to manipulate a regex, so we need
|
||
|
|
# to escape the backslash twice
|
||
|
|
sep = r'\\\\'
|
||
|
|
escaped = r'\1[^%s]' % sep
|
||
|
|
pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
|
||
|
|
return pattern_re
|