628 lines
24 KiB
Python
628 lines
24 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
jinja2.ext
|
||
|
|
~~~~~~~~~~
|
||
|
|
|
||
|
|
Jinja extensions allow to add custom tags similar to the way django custom
|
||
|
|
tags work. By default two example extensions exist: an i18n and a cache
|
||
|
|
extension.
|
||
|
|
|
||
|
|
:copyright: (c) 2017 by the Jinja Team.
|
||
|
|
:license: BSD.
|
||
|
|
"""
|
||
|
|
import re
|
||
|
|
|
||
|
|
from jinja2 import nodes
|
||
|
|
from jinja2.defaults import BLOCK_START_STRING, \
|
||
|
|
BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
|
||
|
|
COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \
|
||
|
|
LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \
|
||
|
|
KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS
|
||
|
|
from jinja2.environment import Environment
|
||
|
|
from jinja2.runtime import concat
|
||
|
|
from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
|
||
|
|
from jinja2.utils import contextfunction, import_string, Markup
|
||
|
|
from jinja2._compat import with_metaclass, string_types, iteritems
|
||
|
|
|
||
|
|
|
||
|
|
# the only real useful gettext functions for a Jinja template. Note
|
||
|
|
# that ugettext must be assigned to gettext as Jinja doesn't support
|
||
|
|
# non unicode strings.
|
||
|
|
GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
|
||
|
|
|
||
|
|
|
||
|
|
class ExtensionRegistry(type):
|
||
|
|
"""Gives the extension an unique identifier."""
|
||
|
|
|
||
|
|
def __new__(cls, name, bases, d):
|
||
|
|
rv = type.__new__(cls, name, bases, d)
|
||
|
|
rv.identifier = rv.__module__ + '.' + rv.__name__
|
||
|
|
return rv
|
||
|
|
|
||
|
|
|
||
|
|
class Extension(with_metaclass(ExtensionRegistry, object)):
|
||
|
|
"""Extensions can be used to add extra functionality to the Jinja template
|
||
|
|
system at the parser level. Custom extensions are bound to an environment
|
||
|
|
but may not store environment specific data on `self`. The reason for
|
||
|
|
this is that an extension can be bound to another environment (for
|
||
|
|
overlays) by creating a copy and reassigning the `environment` attribute.
|
||
|
|
|
||
|
|
As extensions are created by the environment they cannot accept any
|
||
|
|
arguments for configuration. One may want to work around that by using
|
||
|
|
a factory function, but that is not possible as extensions are identified
|
||
|
|
by their import name. The correct way to configure the extension is
|
||
|
|
storing the configuration values on the environment. Because this way the
|
||
|
|
environment ends up acting as central configuration storage the
|
||
|
|
attributes may clash which is why extensions have to ensure that the names
|
||
|
|
they choose for configuration are not too generic. ``prefix`` for example
|
||
|
|
is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
|
||
|
|
name as includes the name of the extension (fragment cache).
|
||
|
|
"""
|
||
|
|
|
||
|
|
#: if this extension parses this is the list of tags it's listening to.
|
||
|
|
tags = set()
|
||
|
|
|
||
|
|
#: the priority of that extension. This is especially useful for
|
||
|
|
#: extensions that preprocess values. A lower value means higher
|
||
|
|
#: priority.
|
||
|
|
#:
|
||
|
|
#: .. versionadded:: 2.4
|
||
|
|
priority = 100
|
||
|
|
|
||
|
|
def __init__(self, environment):
|
||
|
|
self.environment = environment
|
||
|
|
|
||
|
|
def bind(self, environment):
|
||
|
|
"""Create a copy of this extension bound to another environment."""
|
||
|
|
rv = object.__new__(self.__class__)
|
||
|
|
rv.__dict__.update(self.__dict__)
|
||
|
|
rv.environment = environment
|
||
|
|
return rv
|
||
|
|
|
||
|
|
def preprocess(self, source, name, filename=None):
|
||
|
|
"""This method is called before the actual lexing and can be used to
|
||
|
|
preprocess the source. The `filename` is optional. The return value
|
||
|
|
must be the preprocessed source.
|
||
|
|
"""
|
||
|
|
return source
|
||
|
|
|
||
|
|
def filter_stream(self, stream):
|
||
|
|
"""It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
|
||
|
|
to filter tokens returned. This method has to return an iterable of
|
||
|
|
:class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a
|
||
|
|
:class:`~jinja2.lexer.TokenStream`.
|
||
|
|
|
||
|
|
In the `ext` folder of the Jinja2 source distribution there is a file
|
||
|
|
called `inlinegettext.py` which implements a filter that utilizes this
|
||
|
|
method.
|
||
|
|
"""
|
||
|
|
return stream
|
||
|
|
|
||
|
|
def parse(self, parser):
|
||
|
|
"""If any of the :attr:`tags` matched this method is called with the
|
||
|
|
parser as first argument. The token the parser stream is pointing at
|
||
|
|
is the name token that matched. This method has to return one or a
|
||
|
|
list of multiple nodes.
|
||
|
|
"""
|
||
|
|
raise NotImplementedError()
|
||
|
|
|
||
|
|
def attr(self, name, lineno=None):
|
||
|
|
"""Return an attribute node for the current extension. This is useful
|
||
|
|
to pass constants on extensions to generated template code.
|
||
|
|
|
||
|
|
::
|
||
|
|
|
||
|
|
self.attr('_my_attribute', lineno=lineno)
|
||
|
|
"""
|
||
|
|
return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
|
||
|
|
|
||
|
|
def call_method(self, name, args=None, kwargs=None, dyn_args=None,
|
||
|
|
dyn_kwargs=None, lineno=None):
|
||
|
|
"""Call a method of the extension. This is a shortcut for
|
||
|
|
:meth:`attr` + :class:`jinja2.nodes.Call`.
|
||
|
|
"""
|
||
|
|
if args is None:
|
||
|
|
args = []
|
||
|
|
if kwargs is None:
|
||
|
|
kwargs = []
|
||
|
|
return nodes.Call(self.attr(name, lineno=lineno), args, kwargs,
|
||
|
|
dyn_args, dyn_kwargs, lineno=lineno)
|
||
|
|
|
||
|
|
|
||
|
|
@contextfunction
|
||
|
|
def _gettext_alias(__context, *args, **kwargs):
|
||
|
|
return __context.call(__context.resolve('gettext'), *args, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
def _make_new_gettext(func):
|
||
|
|
@contextfunction
|
||
|
|
def gettext(__context, __string, **variables):
|
||
|
|
rv = __context.call(func, __string)
|
||
|
|
if __context.eval_ctx.autoescape:
|
||
|
|
rv = Markup(rv)
|
||
|
|
return rv % variables
|
||
|
|
return gettext
|
||
|
|
|
||
|
|
|
||
|
|
def _make_new_ngettext(func):
|
||
|
|
@contextfunction
|
||
|
|
def ngettext(__context, __singular, __plural, __num, **variables):
|
||
|
|
variables.setdefault('num', __num)
|
||
|
|
rv = __context.call(func, __singular, __plural, __num)
|
||
|
|
if __context.eval_ctx.autoescape:
|
||
|
|
rv = Markup(rv)
|
||
|
|
return rv % variables
|
||
|
|
return ngettext
|
||
|
|
|
||
|
|
|
||
|
|
class InternationalizationExtension(Extension):
|
||
|
|
"""This extension adds gettext support to Jinja2."""
|
||
|
|
tags = set(['trans'])
|
||
|
|
|
||
|
|
# TODO: the i18n extension is currently reevaluating values in a few
|
||
|
|
# situations. Take this example:
|
||
|
|
# {% trans count=something() %}{{ count }} foo{% pluralize
|
||
|
|
# %}{{ count }} fooss{% endtrans %}
|
||
|
|
# something is called twice here. One time for the gettext value and
|
||
|
|
# the other time for the n-parameter of the ngettext function.
|
||
|
|
|
||
|
|
def __init__(self, environment):
|
||
|
|
Extension.__init__(self, environment)
|
||
|
|
environment.globals['_'] = _gettext_alias
|
||
|
|
environment.extend(
|
||
|
|
install_gettext_translations=self._install,
|
||
|
|
install_null_translations=self._install_null,
|
||
|
|
install_gettext_callables=self._install_callables,
|
||
|
|
uninstall_gettext_translations=self._uninstall,
|
||
|
|
extract_translations=self._extract,
|
||
|
|
newstyle_gettext=False
|
||
|
|
)
|
||
|
|
|
||
|
|
def _install(self, translations, newstyle=None):
|
||
|
|
gettext = getattr(translations, 'ugettext', None)
|
||
|
|
if gettext is None:
|
||
|
|
gettext = translations.gettext
|
||
|
|
ngettext = getattr(translations, 'ungettext', None)
|
||
|
|
if ngettext is None:
|
||
|
|
ngettext = translations.ngettext
|
||
|
|
self._install_callables(gettext, ngettext, newstyle)
|
||
|
|
|
||
|
|
def _install_null(self, newstyle=None):
|
||
|
|
self._install_callables(
|
||
|
|
lambda x: x,
|
||
|
|
lambda s, p, n: (n != 1 and (p,) or (s,))[0],
|
||
|
|
newstyle
|
||
|
|
)
|
||
|
|
|
||
|
|
def _install_callables(self, gettext, ngettext, newstyle=None):
|
||
|
|
if newstyle is not None:
|
||
|
|
self.environment.newstyle_gettext = newstyle
|
||
|
|
if self.environment.newstyle_gettext:
|
||
|
|
gettext = _make_new_gettext(gettext)
|
||
|
|
ngettext = _make_new_ngettext(ngettext)
|
||
|
|
self.environment.globals.update(
|
||
|
|
gettext=gettext,
|
||
|
|
ngettext=ngettext
|
||
|
|
)
|
||
|
|
|
||
|
|
def _uninstall(self, translations):
|
||
|
|
for key in 'gettext', 'ngettext':
|
||
|
|
self.environment.globals.pop(key, None)
|
||
|
|
|
||
|
|
def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
|
||
|
|
if isinstance(source, string_types):
|
||
|
|
source = self.environment.parse(source)
|
||
|
|
return extract_from_ast(source, gettext_functions)
|
||
|
|
|
||
|
|
def parse(self, parser):
|
||
|
|
"""Parse a translatable tag."""
|
||
|
|
lineno = next(parser.stream).lineno
|
||
|
|
num_called_num = False
|
||
|
|
|
||
|
|
# find all the variables referenced. Additionally a variable can be
|
||
|
|
# defined in the body of the trans block too, but this is checked at
|
||
|
|
# a later state.
|
||
|
|
plural_expr = None
|
||
|
|
plural_expr_assignment = None
|
||
|
|
variables = {}
|
||
|
|
trimmed = None
|
||
|
|
while parser.stream.current.type != 'block_end':
|
||
|
|
if variables:
|
||
|
|
parser.stream.expect('comma')
|
||
|
|
|
||
|
|
# skip colon for python compatibility
|
||
|
|
if parser.stream.skip_if('colon'):
|
||
|
|
break
|
||
|
|
|
||
|
|
name = parser.stream.expect('name')
|
||
|
|
if name.value in variables:
|
||
|
|
parser.fail('translatable variable %r defined twice.' %
|
||
|
|
name.value, name.lineno,
|
||
|
|
exc=TemplateAssertionError)
|
||
|
|
|
||
|
|
# expressions
|
||
|
|
if parser.stream.current.type == 'assign':
|
||
|
|
next(parser.stream)
|
||
|
|
variables[name.value] = var = parser.parse_expression()
|
||
|
|
elif trimmed is None and name.value in ('trimmed', 'notrimmed'):
|
||
|
|
trimmed = name.value == 'trimmed'
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
variables[name.value] = var = nodes.Name(name.value, 'load')
|
||
|
|
|
||
|
|
if plural_expr is None:
|
||
|
|
if isinstance(var, nodes.Call):
|
||
|
|
plural_expr = nodes.Name('_trans', 'load')
|
||
|
|
variables[name.value] = plural_expr
|
||
|
|
plural_expr_assignment = nodes.Assign(
|
||
|
|
nodes.Name('_trans', 'store'), var)
|
||
|
|
else:
|
||
|
|
plural_expr = var
|
||
|
|
num_called_num = name.value == 'num'
|
||
|
|
|
||
|
|
parser.stream.expect('block_end')
|
||
|
|
|
||
|
|
plural = None
|
||
|
|
have_plural = False
|
||
|
|
referenced = set()
|
||
|
|
|
||
|
|
# now parse until endtrans or pluralize
|
||
|
|
singular_names, singular = self._parse_block(parser, True)
|
||
|
|
if singular_names:
|
||
|
|
referenced.update(singular_names)
|
||
|
|
if plural_expr is None:
|
||
|
|
plural_expr = nodes.Name(singular_names[0], 'load')
|
||
|
|
num_called_num = singular_names[0] == 'num'
|
||
|
|
|
||
|
|
# if we have a pluralize block, we parse that too
|
||
|
|
if parser.stream.current.test('name:pluralize'):
|
||
|
|
have_plural = True
|
||
|
|
next(parser.stream)
|
||
|
|
if parser.stream.current.type != 'block_end':
|
||
|
|
name = parser.stream.expect('name')
|
||
|
|
if name.value not in variables:
|
||
|
|
parser.fail('unknown variable %r for pluralization' %
|
||
|
|
name.value, name.lineno,
|
||
|
|
exc=TemplateAssertionError)
|
||
|
|
plural_expr = variables[name.value]
|
||
|
|
num_called_num = name.value == 'num'
|
||
|
|
parser.stream.expect('block_end')
|
||
|
|
plural_names, plural = self._parse_block(parser, False)
|
||
|
|
next(parser.stream)
|
||
|
|
referenced.update(plural_names)
|
||
|
|
else:
|
||
|
|
next(parser.stream)
|
||
|
|
|
||
|
|
# register free names as simple name expressions
|
||
|
|
for var in referenced:
|
||
|
|
if var not in variables:
|
||
|
|
variables[var] = nodes.Name(var, 'load')
|
||
|
|
|
||
|
|
if not have_plural:
|
||
|
|
plural_expr = None
|
||
|
|
elif plural_expr is None:
|
||
|
|
parser.fail('pluralize without variables', lineno)
|
||
|
|
|
||
|
|
if trimmed is None:
|
||
|
|
trimmed = self.environment.policies['ext.i18n.trimmed']
|
||
|
|
if trimmed:
|
||
|
|
singular = self._trim_whitespace(singular)
|
||
|
|
if plural:
|
||
|
|
plural = self._trim_whitespace(plural)
|
||
|
|
|
||
|
|
node = self._make_node(singular, plural, variables, plural_expr,
|
||
|
|
bool(referenced),
|
||
|
|
num_called_num and have_plural)
|
||
|
|
node.set_lineno(lineno)
|
||
|
|
if plural_expr_assignment is not None:
|
||
|
|
return [plural_expr_assignment, node]
|
||
|
|
else:
|
||
|
|
return node
|
||
|
|
|
||
|
|
def _trim_whitespace(self, string, _ws_re=re.compile(r'\s*\n\s*')):
|
||
|
|
return _ws_re.sub(' ', string.strip())
|
||
|
|
|
||
|
|
def _parse_block(self, parser, allow_pluralize):
|
||
|
|
"""Parse until the next block tag with a given name."""
|
||
|
|
referenced = []
|
||
|
|
buf = []
|
||
|
|
while 1:
|
||
|
|
if parser.stream.current.type == 'data':
|
||
|
|
buf.append(parser.stream.current.value.replace('%', '%%'))
|
||
|
|
next(parser.stream)
|
||
|
|
elif parser.stream.current.type == 'variable_begin':
|
||
|
|
next(parser.stream)
|
||
|
|
name = parser.stream.expect('name').value
|
||
|
|
referenced.append(name)
|
||
|
|
buf.append('%%(%s)s' % name)
|
||
|
|
parser.stream.expect('variable_end')
|
||
|
|
elif parser.stream.current.type == 'block_begin':
|
||
|
|
next(parser.stream)
|
||
|
|
if parser.stream.current.test('name:endtrans'):
|
||
|
|
break
|
||
|
|
elif parser.stream.current.test('name:pluralize'):
|
||
|
|
if allow_pluralize:
|
||
|
|
break
|
||
|
|
parser.fail('a translatable section can have only one '
|
||
|
|
'pluralize section')
|
||
|
|
parser.fail('control structures in translatable sections are '
|
||
|
|
'not allowed')
|
||
|
|
elif parser.stream.eos:
|
||
|
|
parser.fail('unclosed translation block')
|
||
|
|
else:
|
||
|
|
assert False, 'internal parser error'
|
||
|
|
|
||
|
|
return referenced, concat(buf)
|
||
|
|
|
||
|
|
def _make_node(self, singular, plural, variables, plural_expr,
|
||
|
|
vars_referenced, num_called_num):
|
||
|
|
"""Generates a useful node from the data provided."""
|
||
|
|
# no variables referenced? no need to escape for old style
|
||
|
|
# gettext invocations only if there are vars.
|
||
|
|
if not vars_referenced and not self.environment.newstyle_gettext:
|
||
|
|
singular = singular.replace('%%', '%')
|
||
|
|
if plural:
|
||
|
|
plural = plural.replace('%%', '%')
|
||
|
|
|
||
|
|
# singular only:
|
||
|
|
if plural_expr is None:
|
||
|
|
gettext = nodes.Name('gettext', 'load')
|
||
|
|
node = nodes.Call(gettext, [nodes.Const(singular)],
|
||
|
|
[], None, None)
|
||
|
|
|
||
|
|
# singular and plural
|
||
|
|
else:
|
||
|
|
ngettext = nodes.Name('ngettext', 'load')
|
||
|
|
node = nodes.Call(ngettext, [
|
||
|
|
nodes.Const(singular),
|
||
|
|
nodes.Const(plural),
|
||
|
|
plural_expr
|
||
|
|
], [], None, None)
|
||
|
|
|
||
|
|
# in case newstyle gettext is used, the method is powerful
|
||
|
|
# enough to handle the variable expansion and autoescape
|
||
|
|
# handling itself
|
||
|
|
if self.environment.newstyle_gettext:
|
||
|
|
for key, value in iteritems(variables):
|
||
|
|
# the function adds that later anyways in case num was
|
||
|
|
# called num, so just skip it.
|
||
|
|
if num_called_num and key == 'num':
|
||
|
|
continue
|
||
|
|
node.kwargs.append(nodes.Keyword(key, value))
|
||
|
|
|
||
|
|
# otherwise do that here
|
||
|
|
else:
|
||
|
|
# mark the return value as safe if we are in an
|
||
|
|
# environment with autoescaping turned on
|
||
|
|
node = nodes.MarkSafeIfAutoescape(node)
|
||
|
|
if variables:
|
||
|
|
node = nodes.Mod(node, nodes.Dict([
|
||
|
|
nodes.Pair(nodes.Const(key), value)
|
||
|
|
for key, value in variables.items()
|
||
|
|
]))
|
||
|
|
return nodes.Output([node])
|
||
|
|
|
||
|
|
|
||
|
|
class ExprStmtExtension(Extension):
|
||
|
|
"""Adds a `do` tag to Jinja2 that works like the print statement just
|
||
|
|
that it doesn't print the return value.
|
||
|
|
"""
|
||
|
|
tags = set(['do'])
|
||
|
|
|
||
|
|
def parse(self, parser):
|
||
|
|
node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
|
||
|
|
node.node = parser.parse_tuple()
|
||
|
|
return node
|
||
|
|
|
||
|
|
|
||
|
|
class LoopControlExtension(Extension):
|
||
|
|
"""Adds break and continue to the template engine."""
|
||
|
|
tags = set(['break', 'continue'])
|
||
|
|
|
||
|
|
def parse(self, parser):
|
||
|
|
token = next(parser.stream)
|
||
|
|
if token.value == 'break':
|
||
|
|
return nodes.Break(lineno=token.lineno)
|
||
|
|
return nodes.Continue(lineno=token.lineno)
|
||
|
|
|
||
|
|
|
||
|
|
class WithExtension(Extension):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
class AutoEscapeExtension(Extension):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
|
||
|
|
babel_style=True):
|
||
|
|
"""Extract localizable strings from the given template node. Per
|
||
|
|
default this function returns matches in babel style that means non string
|
||
|
|
parameters as well as keyword arguments are returned as `None`. This
|
||
|
|
allows Babel to figure out what you really meant if you are using
|
||
|
|
gettext functions that allow keyword arguments for placeholder expansion.
|
||
|
|
If you don't want that behavior set the `babel_style` parameter to `False`
|
||
|
|
which causes only strings to be returned and parameters are always stored
|
||
|
|
in tuples. As a consequence invalid gettext calls (calls without a single
|
||
|
|
string parameter or string parameters after non-string parameters) are
|
||
|
|
skipped.
|
||
|
|
|
||
|
|
This example explains the behavior:
|
||
|
|
|
||
|
|
>>> from jinja2 import Environment
|
||
|
|
>>> env = Environment()
|
||
|
|
>>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
|
||
|
|
>>> list(extract_from_ast(node))
|
||
|
|
[(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
|
||
|
|
>>> list(extract_from_ast(node, babel_style=False))
|
||
|
|
[(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
|
||
|
|
|
||
|
|
For every string found this function yields a ``(lineno, function,
|
||
|
|
message)`` tuple, where:
|
||
|
|
|
||
|
|
* ``lineno`` is the number of the line on which the string was found,
|
||
|
|
* ``function`` is the name of the ``gettext`` function used (if the
|
||
|
|
string was extracted from embedded Python code), and
|
||
|
|
* ``message`` is the string itself (a ``unicode`` object, or a tuple
|
||
|
|
of ``unicode`` objects for functions with multiple string arguments).
|
||
|
|
|
||
|
|
This extraction function operates on the AST and is because of that unable
|
||
|
|
to extract any comments. For comment support you have to use the babel
|
||
|
|
extraction interface or extract comments yourself.
|
||
|
|
"""
|
||
|
|
for node in node.find_all(nodes.Call):
|
||
|
|
if not isinstance(node.node, nodes.Name) or \
|
||
|
|
node.node.name not in gettext_functions:
|
||
|
|
continue
|
||
|
|
|
||
|
|
strings = []
|
||
|
|
for arg in node.args:
|
||
|
|
if isinstance(arg, nodes.Const) and \
|
||
|
|
isinstance(arg.value, string_types):
|
||
|
|
strings.append(arg.value)
|
||
|
|
else:
|
||
|
|
strings.append(None)
|
||
|
|
|
||
|
|
for arg in node.kwargs:
|
||
|
|
strings.append(None)
|
||
|
|
if node.dyn_args is not None:
|
||
|
|
strings.append(None)
|
||
|
|
if node.dyn_kwargs is not None:
|
||
|
|
strings.append(None)
|
||
|
|
|
||
|
|
if not babel_style:
|
||
|
|
strings = tuple(x for x in strings if x is not None)
|
||
|
|
if not strings:
|
||
|
|
continue
|
||
|
|
else:
|
||
|
|
if len(strings) == 1:
|
||
|
|
strings = strings[0]
|
||
|
|
else:
|
||
|
|
strings = tuple(strings)
|
||
|
|
yield node.lineno, node.node.name, strings
|
||
|
|
|
||
|
|
|
||
|
|
class _CommentFinder(object):
|
||
|
|
"""Helper class to find comments in a token stream. Can only
|
||
|
|
find comments for gettext calls forwards. Once the comment
|
||
|
|
from line 4 is found, a comment for line 1 will not return a
|
||
|
|
usable value.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, tokens, comment_tags):
|
||
|
|
self.tokens = tokens
|
||
|
|
self.comment_tags = comment_tags
|
||
|
|
self.offset = 0
|
||
|
|
self.last_lineno = 0
|
||
|
|
|
||
|
|
def find_backwards(self, offset):
|
||
|
|
try:
|
||
|
|
for _, token_type, token_value in \
|
||
|
|
reversed(self.tokens[self.offset:offset]):
|
||
|
|
if token_type in ('comment', 'linecomment'):
|
||
|
|
try:
|
||
|
|
prefix, comment = token_value.split(None, 1)
|
||
|
|
except ValueError:
|
||
|
|
continue
|
||
|
|
if prefix in self.comment_tags:
|
||
|
|
return [comment.rstrip()]
|
||
|
|
return []
|
||
|
|
finally:
|
||
|
|
self.offset = offset
|
||
|
|
|
||
|
|
def find_comments(self, lineno):
|
||
|
|
if not self.comment_tags or self.last_lineno > lineno:
|
||
|
|
return []
|
||
|
|
for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
|
||
|
|
if token_lineno > lineno:
|
||
|
|
return self.find_backwards(self.offset + idx)
|
||
|
|
return self.find_backwards(len(self.tokens))
|
||
|
|
|
||
|
|
|
||
|
|
def babel_extract(fileobj, keywords, comment_tags, options):
|
||
|
|
"""Babel extraction method for Jinja templates.
|
||
|
|
|
||
|
|
.. versionchanged:: 2.3
|
||
|
|
Basic support for translation comments was added. If `comment_tags`
|
||
|
|
is now set to a list of keywords for extraction, the extractor will
|
||
|
|
try to find the best preceeding comment that begins with one of the
|
||
|
|
keywords. For best results, make sure to not have more than one
|
||
|
|
gettext call in one line of code and the matching comment in the
|
||
|
|
same line or the line before.
|
||
|
|
|
||
|
|
.. versionchanged:: 2.5.1
|
||
|
|
The `newstyle_gettext` flag can be set to `True` to enable newstyle
|
||
|
|
gettext calls.
|
||
|
|
|
||
|
|
.. versionchanged:: 2.7
|
||
|
|
A `silent` option can now be provided. If set to `False` template
|
||
|
|
syntax errors are propagated instead of being ignored.
|
||
|
|
|
||
|
|
:param fileobj: the file-like object the messages should be extracted from
|
||
|
|
:param keywords: a list of keywords (i.e. function names) that should be
|
||
|
|
recognized as translation functions
|
||
|
|
:param comment_tags: a list of translator tags to search for and include
|
||
|
|
in the results.
|
||
|
|
:param options: a dictionary of additional options (optional)
|
||
|
|
:return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
|
||
|
|
(comments will be empty currently)
|
||
|
|
"""
|
||
|
|
extensions = set()
|
||
|
|
for extension in options.get('extensions', '').split(','):
|
||
|
|
extension = extension.strip()
|
||
|
|
if not extension:
|
||
|
|
continue
|
||
|
|
extensions.add(import_string(extension))
|
||
|
|
if InternationalizationExtension not in extensions:
|
||
|
|
extensions.add(InternationalizationExtension)
|
||
|
|
|
||
|
|
def getbool(options, key, default=False):
|
||
|
|
return options.get(key, str(default)).lower() in \
|
||
|
|
('1', 'on', 'yes', 'true')
|
||
|
|
|
||
|
|
silent = getbool(options, 'silent', True)
|
||
|
|
environment = Environment(
|
||
|
|
options.get('block_start_string', BLOCK_START_STRING),
|
||
|
|
options.get('block_end_string', BLOCK_END_STRING),
|
||
|
|
options.get('variable_start_string', VARIABLE_START_STRING),
|
||
|
|
options.get('variable_end_string', VARIABLE_END_STRING),
|
||
|
|
options.get('comment_start_string', COMMENT_START_STRING),
|
||
|
|
options.get('comment_end_string', COMMENT_END_STRING),
|
||
|
|
options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
|
||
|
|
options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
|
||
|
|
getbool(options, 'trim_blocks', TRIM_BLOCKS),
|
||
|
|
getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS),
|
||
|
|
NEWLINE_SEQUENCE,
|
||
|
|
getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE),
|
||
|
|
frozenset(extensions),
|
||
|
|
cache_size=0,
|
||
|
|
auto_reload=False
|
||
|
|
)
|
||
|
|
|
||
|
|
if getbool(options, 'trimmed'):
|
||
|
|
environment.policies['ext.i18n.trimmed'] = True
|
||
|
|
if getbool(options, 'newstyle_gettext'):
|
||
|
|
environment.newstyle_gettext = True
|
||
|
|
|
||
|
|
source = fileobj.read().decode(options.get('encoding', 'utf-8'))
|
||
|
|
try:
|
||
|
|
node = environment.parse(source)
|
||
|
|
tokens = list(environment.lex(environment.preprocess(source)))
|
||
|
|
except TemplateSyntaxError as e:
|
||
|
|
if not silent:
|
||
|
|
raise
|
||
|
|
# skip templates with syntax errors
|
||
|
|
return
|
||
|
|
|
||
|
|
finder = _CommentFinder(tokens, comment_tags)
|
||
|
|
for lineno, func, message in extract_from_ast(node, keywords):
|
||
|
|
yield lineno, func, message, finder.find_comments(lineno)
|
||
|
|
|
||
|
|
|
||
|
|
#: nicer import names
|
||
|
|
i18n = InternationalizationExtension
|
||
|
|
do = ExprStmtExtension
|
||
|
|
loopcontrols = LoopControlExtension
|
||
|
|
with_ = WithExtension
|
||
|
|
autoescape = AutoEscapeExtension
|