diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 681508eedb6fbb1b044df8d813203976720adc76..e3d5a638131c90ef70213c0191c34f0175c2b784 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -2,7 +2,7 @@ name: Tests
on: [push, pull_request]
jobs:
build:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev]
diff --git a/make_release.sh b/make_release.sh
index f0c1ac84a64f29c6b0af412ba2f0f4bc7d55a9e0..0415e1d35e3066592514d0866719686e4adeab7c 100755
--- a/make_release.sh
+++ b/make_release.sh
@@ -20,7 +20,7 @@ else
fi
fi
-tox -p auto
+tox -p 3
export TAG="v${1}"
git tag "${TAG}"
diff --git a/setup.cfg b/setup.cfg
index dc8b29ba3a133b5f0fccae8b5fea81e02e347961..05f3ddac28c780df1ae323fe2e072d76c409e77a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -22,7 +22,11 @@ classifiers =
[options]
packages = stack_data
-install_requires = executing; asttokens; pure_eval
+install_requires =
+ executing>=1.2.0
+ asttokens>=2.1.0
+ pure_eval
+
setup_requires = setuptools>=44; setuptools_scm[toml]>=3.4.3
include_package_data = True
tests_require = pytest; typeguard; pygments; littleutils
diff --git a/stack_data/core.py b/stack_data/core.py
index e2ca2d5a246613fe6d4cc050b6b14000d0d682ce..88e060392adc5d713b5593302cbfb01023a37613 100644
--- a/stack_data/core.py
+++ b/stack_data/core.py
@@ -91,11 +91,6 @@ class Source(executing.Source):
Don't construct this class. Get an instance from frame_info.source.
"""
- def __init__(self, *args, **kwargs):
- super(Source, self).__init__(*args, **kwargs)
- if self.tree:
- self.asttokens()
-
@cached_property
def pieces(self) -> List[range]:
if not self.tree:
@@ -122,6 +117,20 @@ class Source(executing.Source):
if end > start
]
+ # Combine overlapping pieces, i.e. consecutive pieces where the end of the first
+ # is greater than the start of the second.
+ # This can happen when two statements are on the same line separated by a semicolon.
+ new_pieces = pieces[:1]
+ for (start, end) in pieces[1:]:
+ (last_start, last_end) = new_pieces[-1]
+ if start < last_end:
+ assert start == last_end - 1
+ assert ';' in self.lines[start - 1]
+ new_pieces[-1] = (last_start, end)
+ else:
+ new_pieces.append((start, end))
+ pieces = new_pieces
+
starts = [start for start, end in pieces[1:]]
ends = [end for start, end in pieces[:-1]]
if starts != ends:
@@ -149,14 +158,12 @@ class Source(executing.Source):
start: int,
end: int,
) -> Iterator[Tuple[int, int]]:
- self.asttokens()
-
for name, body in ast.iter_fields(stmt):
if (
isinstance(body, list) and body and
isinstance(body[0], (ast.stmt, ast.ExceptHandler, getattr(ast, 'match_case', ())))
):
- for rang, group in sorted(group_by_key_func(body, line_range).items()):
+ for rang, group in sorted(group_by_key_func(body, self.line_range).items()):
sub_stmt = group[0]
for inner_start, inner_end in self._raw_split_into_pieces(sub_stmt, *rang):
if start < inner_start:
@@ -167,6 +174,9 @@ class Source(executing.Source):
yield start, end
+ def line_range(self, node: ast.AST) -> Tuple[int, int]:
+ return line_range(self.asttext(), node)
+
class Options:
"""
@@ -342,28 +352,18 @@ class Line(object):
with the correct start and end and the given data.
Otherwise, return None.
"""
- start, end = line_range(node)
- end -= 1
+ atext = self.frame_info.source.asttext()
+ (start, range_start), (end, range_end) = atext.get_text_positions(node, padded=False)
+
if not (start <= self.lineno <= end):
return None
- if start == self.lineno:
- try:
- range_start = node.first_token.start[1]
- except AttributeError:
- range_start = node.col_offset
- else:
+
+ if start != self.lineno:
range_start = common_indent
- if end == self.lineno:
- try:
- range_end = node.last_token.end[1]
- except AttributeError:
- try:
- range_end = node.end_col_offset
- except AttributeError:
- return None
- else:
+ if end != self.lineno:
range_end = len(self.text)
+
if range_start == range_end == 0:
# This is an empty line. If it were included, it would result
# in a value of zero for the common indentation assigned to
@@ -611,7 +611,7 @@ class FrameInfo(object):
if not self.scope:
return self.source.pieces
- scope_start, scope_end = line_range(self.scope)
+ scope_start, scope_end = self.source.line_range(self.scope)
return [
piece
for piece in self.source.pieces
@@ -809,21 +809,21 @@ class FrameInfo(object):
if isinstance(formatter, HtmlFormatter):
formatter.nowrap = True
- atok = self.source.asttokens()
+ atext = self.source.asttext()
node = self.executing.node
if node and getattr(formatter.style, "for_executing_node", False):
- scope_start = atok.get_text_range(scope)[0]
- start, end = atok.get_text_range(node)
+ scope_start = atext.get_text_range(scope)[0]
+ start, end = atext.get_text_range(node)
start -= scope_start
end -= scope_start
ranges = [(start, end)]
else:
ranges = []
- code = atok.get_text(scope)
+ code = atext.get_text(scope)
lines = _pygmented_with_ranges(formatter, code, ranges)
- start_line = line_range(scope)[0]
+ start_line = self.source.line_range(scope)[0]
return start_line, lines
@@ -861,7 +861,7 @@ class FrameInfo(object):
if isinstance(n, ast.arg):
return n.arg
else:
- return self.source.asttokens().get_text(n)
+ return self.source.asttext().get_text(n)
def normalise_node(n):
try:
@@ -897,7 +897,7 @@ class FrameInfo(object):
result = defaultdict(list)
for var in self.variables:
for node in var.nodes:
- for lineno in range(*line_range(node)):
+ for lineno in range(*self.source.line_range(node)):
result[lineno].append((var, node))
return result
diff --git a/stack_data/utils.py b/stack_data/utils.py
index 9cd0633c56d3a47171266487386700ad68a0fba9..78ce2d60a400b0acb3753ca038412ed1b66d0199 100644
--- a/stack_data/utils.py
+++ b/stack_data/utils.py
@@ -8,6 +8,8 @@ from typing import (
TypeVar, Mapping,
)
+from asttokens import ASTText
+
T = TypeVar('T')
R = TypeVar('R')
@@ -24,22 +26,19 @@ def unique_in_order(it: Iterable[T]) -> List[T]:
return list(OrderedDict.fromkeys(it))
-def line_range(node: ast.AST) -> Tuple[int, int]:
+def line_range(atok: ASTText, node: ast.AST) -> Tuple[int, int]:
"""
Returns a pair of numbers representing a half open range
(i.e. suitable as arguments to the `range()` builtin)
of line numbers of the given AST nodes.
"""
- try:
- return (
- node.first_token.start[0],
- node.last_token.end[0] + 1,
- )
- except AttributeError:
- return (
- node.lineno,
- getattr(node, "end_lineno", node.lineno) + 1,
- )
+ if isinstance(node, getattr(ast, "match_case", ())):
+ start, _end = line_range(atok, node.pattern)
+ _start, end = line_range(atok, node.body[-1])
+ return start, end
+ else:
+ (start, _), (end, _) = atok.get_text_positions(node, padded=False)
+ return start, end + 1
def highlight_unique(lst: List[T]) -> Iterator[Tuple[T, bool]]:
@@ -162,7 +161,12 @@ def _pygmented_with_ranges(formatter, code, ranges):
yield ttype, value
lexer = MyLexer(stripnl=False)
- return pygments.highlight(code, lexer, formatter).splitlines()
+ try:
+ highlighted = pygments.highlight(code, lexer, formatter)
+ except Exception:
+ # When pygments fails, prefer code without highlighting over crashing
+ highlighted = code
+ return highlighted.splitlines()
def assert_(condition, error=""):
diff --git a/tests/golden_files/f_string_3.8.txt b/tests/golden_files/f_string_3.8.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e9d447a74d284a0f07b62d1ab1ff7e214d7e8d7a
--- /dev/null
+++ b/tests/golden_files/f_string_3.8.txt
@@ -0,0 +1,11 @@
+Traceback (most recent call last):
+ File "formatter_example.py", line 57, in f_string
+ 54 | def f_string():
+ 55 | f"""{str
+ 56 | (
+--> 57 | 1 /
+ 58 | 0 + 4
+ 59 | + 5
+ 60 | )
+ 61 | }"""
+ZeroDivisionError: division by zero
diff --git a/tests/golden_files/pygmented_error.txt b/tests/golden_files/pygmented_error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3971566b26fcadfd5909f14e2e3f0c7a0368db57
--- /dev/null
+++ b/tests/golden_files/pygmented_error.txt
@@ -0,0 +1,54 @@
+Traceback (most recent call last):
+ File "formatter_example.py", line 21, in foo
+ 9 | x = 1
+ 10 | lst = (
+ 11 | [
+ 12 | x,
+(...)
+ 18 | + []
+ 19 | )
+ 20 | try:
+--> 21 | return int(str(lst))
+ 22 | except:
+ValueError: invalid literal for int() with base 10: '[1]'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "formatter_example.py", line 24, in foo
+ 21 | return int(str(lst))
+ 22 | except:
+ 23 | try:
+--> 24 | return 1 / 0
+ 25 | except Exception as e:
+ZeroDivisionError: division by zero
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+ File "formatter_example.py", line 30, in bar
+ 29 | def bar():
+--> 30 | exec("foo()")
+ File "<string>", line 1, in <module>
+ File "formatter_example.py", line 8, in foo
+ 6 | def foo(n=5):
+ 7 | if n > 0:
+--> 8 | return foo(n - 1)
+ 9 | x = 1
+ File "formatter_example.py", line 8, in foo
+ 6 | def foo(n=5):
+ 7 | if n > 0:
+--> 8 | return foo(n - 1)
+ 9 | x = 1
+ [... skipping similar frames: foo at line 8 (2 times)]
+ File "formatter_example.py", line 8, in foo
+ 6 | def foo(n=5):
+ 7 | if n > 0:
+--> 8 | return foo(n - 1)
+ 9 | x = 1
+ File "formatter_example.py", line 26, in foo
+ 23 | try:
+ 24 | return 1 / 0
+ 25 | except Exception as e:
+--> 26 | raise TypeError from e
+TypeError
diff --git a/tests/samples/pieces.py b/tests/samples/pieces.py
index d61b899ecb9892742f33e02fc1c1fa265a19e0ea..cc34a36a41cc91a569f42cef0fcb942a7f3e4e1e 100644
--- a/tests/samples/pieces.py
+++ b/tests/samples/pieces.py
@@ -79,3 +79,19 @@ class Foo(object):
@property
def foo(self):
return 3
+
+
+# noinspection PyTrailingSemicolon
+def semicolons():
+ if 1:
+ print(1,
+ 2); print(3,
+ 4); print(5,
+ 6)
+ if 2:
+ print(1,
+ 2); print(3, 4); print(5,
+ 6)
+ print(1, 2); print(3,
+ 4); print(5, 6)
+ print(1, 2);print(3, 4);print(5, 6)
diff --git a/tests/test_core.py b/tests/test_core.py
index 870497dda487db9d2ea40c9a734acd0dc0ee9ea7..52ad3aeac1aa8f1a33cc4565d49e9b6b39984404 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -399,7 +399,21 @@ def test_pieces():
['class Foo(object):'],
[' @property',
' def foo(self):'],
- [' return 3']
+ [' return 3'],
+ ['# noinspection PyTrailingSemicolon'],
+ ['def semicolons():'],
+ [' if 1:'],
+ [' print(1,',
+ ' 2); print(3,',
+ ' 4); print(5,',
+ ' 6)'],
+ [' if 2:'],
+ [' print(1,',
+ ' 2); print(3, 4); print(5,',
+ ' 6)'],
+ [' print(1, 2); print(3,',
+ ' 4); print(5, 6)'],
+ [' print(1, 2);print(3, 4);print(5, 6)']
]
@@ -501,7 +515,7 @@ def check_pieces(source):
assert pieces == sorted(pieces, key=lambda p: (p.start, p.stop))
stmts = sorted({
- line_range(node)
+ source.line_range(node)
for node in ast.walk(source.tree)
if isinstance(node, ast.stmt)
if not isinstance(getattr(node, 'body', None), list)
diff --git a/tests/test_formatter.py b/tests/test_formatter.py
index 1e40b68008ba3b017d3c16d3d1ad3ce005cb4516..31862f82e44ac409d3f06be641e897c462a34a5a 100644
--- a/tests/test_formatter.py
+++ b/tests/test_formatter.py
@@ -4,6 +4,7 @@ import sys
from contextlib import contextmanager
import pytest
+import pygments
from stack_data import Formatter, FrameInfo, Options, BlankLines
from tests.utils import compare_to_file
@@ -56,6 +57,16 @@ def test_example(capsys):
except Exception:
sys.excepthook(*sys.exc_info())
+ with check_example("pygmented_error"):
+ h = pygments.highlight
+ pygments.highlight = lambda *args, **kwargs: 1/0
+ try:
+ bar()
+ except Exception:
+ MyFormatter(pygmented=True).print_exception()
+ finally:
+ pygments.highlight = h
+
with check_example("print_stack"):
print_stack1(MyFormatter())
@@ -69,7 +80,15 @@ def test_example(capsys):
formatted = format_frame(formatter)
formatter.print_lines(formatted)
- with check_example(f"f_string_{'old' if sys.version_info[:2] < (3, 8) else 'new'}"):
+ if sys.version_info[:2] < (3, 8):
+ f_string_suffix = 'old'
+ elif sys.version_info[:2] == (3, 8):
+ # lineno/col_offset in f-strings cannot be trusted in 3.8
+ f_string_suffix = '3.8'
+ else:
+ f_string_suffix = 'new'
+
+ with check_example(f"f_string_{f_string_suffix}"):
try:
f_string()
except Exception: