:mod:`xdoctest.static_analysis`
===============================

.. py:module:: xdoctest.static_analysis

.. autoapi-nested-parse::

   The core logic that allows for xdoctest to parse source statically



Module Contents
---------------

Classes
~~~~~~~

.. autoapisummary::

   xdoctest.static_analysis.CallDefNode
   xdoctest.static_analysis.TopLevelVisitor



Functions
~~~~~~~~~

.. autoapisummary::

   xdoctest.static_analysis.parse_static_calldefs
   xdoctest.static_analysis.parse_calldefs
   xdoctest.static_analysis._parse_static_node_value
   xdoctest.static_analysis.parse_static_value
   xdoctest.static_analysis.package_modpaths
   xdoctest.static_analysis.is_balanced_statement
   xdoctest.static_analysis.extract_comments
   xdoctest.static_analysis.six_axt_parse


.. data:: PLAT_IMPL
   

   

.. data:: HAS_UPDATED_LINENOS
   

   

.. py:class:: CallDefNode(callname, lineno, docstr, doclineno, doclineno_end, args=None)

   Bases: :class:`object`

   :ivar callname: the name of the "calldef"
   :vartype callname: str
   :ivar doclineno: the line number (1 based) the docstring begins on
   :vartype doclineno: int
   :ivar doclineno_end: the line number (1 based) the docstring ends on

   :vartype doclineno_end: int

   .. method:: __str__(self)


      Return str(self).



.. py:class:: TopLevelVisitor(source=None)

   Bases: :class:`ast.NodeVisitor`

   Parses top-level function names and docstrings

   .. rubric:: References

   # For other visit_<classname> values see
   http://greentreesnakes.readthedocs.io/en/latest/nodes.html

   CommandLine:
       python -m xdoctest.static_analysis TopLevelVisitor

   .. rubric:: Example

   >>> from xdoctest.static_analysis import *  # NOQA
   >>> from xdoctest import utils
   >>> source = utils.codeblock(
           '''
           def foo():
               """ my docstring """
               def subfunc():
                   pass
           def bar():
               pass
           class Spam(object):
               def eggs(self):
                   pass
               @staticmethod
               def hams():
                   pass
               @property
               def jams(self):
                   return 3
               @jams.setter
               def jams2(self, x):
                   print('ignoring')
               @jams.deleter
               def jams(self, x):
                   print('ignoring')
           ''')
   >>> self = TopLevelVisitor.parse(source)
   >>> callnames = set(self.calldefs.keys())
   >>> assert callnames == {
   >>>     'foo', 'bar', 'Spam', 'Spam.eggs', 'Spam.hams',
   >>>     'Spam.jams'}
   >>> assert self.calldefs['foo'].docstr.strip() == 'my docstring'
   >>> assert 'subfunc' not in self.calldefs

   .. method:: parse(cls, source)
      :classmethod:


      main entry point

      executes parsing algorithm and populates self.calldefs


   .. method:: syntax_tree(self)


      creates the abstract syntax tree


   .. method:: process_finished(self, node)


      process (get ending lineno) for everything marked as finished


   .. method:: visit(self, node)


      Visit a node.


   .. method:: visit_FunctionDef(self, node)



   .. method:: visit_ClassDef(self, node)



   .. method:: visit_Module(self, node)



   .. method:: visit_Assign(self, node)



   .. method:: visit_If(self, node)



   .. method:: _docnode_line_workaround(self, docnode)


      Find the start and ending line numbers of a docstring

      CommandLine:
          xdoctest -m xdoctest.static_analysis TopLevelVisitor._docnode_line_workaround

      .. rubric:: Example

      >>> from xdoctest.static_analysis import *  # NOQA
      >>> sq = chr(39)  # single quote
      >>> dq = chr(34)  # double quote
      >>> source = utils.codeblock(
          '''
          def func0():
              {ddd} docstr0 {ddd}
          def func1():
              {ddd}
              docstr1 {ddd}
          def func2():
              {ddd} docstr2
              {ddd}
          def func3():
              {ddd}
              docstr3
              {ddd}  # foobar
          def func5():
              {ddd}pathological case
              {sss} # {ddd} # {sss} # {ddd} # {ddd}
          def func6():
              " single quoted docstr "
          def func7():
              r{ddd}
              raw line
              {ddd}
          ''').format(ddd=dq * 3, sss=sq * 3)
      >>> self = TopLevelVisitor(source)
      >>> func_nodes = self.syntax_tree().body
      >>> print(utils.add_line_numbers(utils.highlight_code(source), start=1))
      >>> wants = [
      >>>     (2, 2),
      >>>     (4, 5),
      >>>     (7, 8),
      >>>     (10, 12),
      >>>     (14, 15),
      >>>     (17, 17),
      >>>     (19, 21),
      >>> ]
      >>> for i, func_node in enumerate(func_nodes):
      >>>     docnode = func_node.body[0]
      >>>     got = self._docnode_line_workaround(docnode)
      >>>     want = wants[i]
      >>>     print('got = {!r}'.format(got))
      >>>     print('want = {!r}'.format(want))
      >>>     assert got == want


   .. method:: _find_docstr_endpos_workaround(cls, docstr, sourcelines, startpos)
      :classmethod:


      Like docstr_line_workaround, but works from the top-down instead of
      bottom-up. This is for pypy.


      Given a docstring, its original source lines, and where the start
      position is, this function finds the end-position of the docstr

      .. rubric:: Example

      >>> fmtkw = dict(sss=chr(39) * 3, ddd=chr(34) * 3)
      >>> source = utils.codeblock(
              '''
              {ddd}
              docstr0
              {ddd}
              '''.format(**fmtkw))
      >>> sourcelines = source.splitlines()
      >>> docstr = eval(source, {}, {})
      >>> startpos = 0
      >>> start, stop = TopLevelVisitor._find_docstr_endpos_workaround(docstr, sourcelines, startpos)
      >>> assert (start, stop) == (0, 2)
      >>> #
      >>> source = utils.codeblock(
              '''
              "docstr0"
              '''.format(**fmtkw))
      >>> sourcelines = source.splitlines()
      >>> docstr = eval(source, {}, {})
      >>> startpos = 0
      >>> start, stop = TopLevelVisitor._find_docstr_endpos_workaround(docstr, sourcelines, startpos)
      >>> assert (start, stop) == (0, 0)


   .. method:: _find_docstr_startpos_workaround(self, docstr, sourcelines, endpos)


      Find the which sourcelines contain the docstring

      :Parameters: * **docstr** (*str*) -- the extracted docstring.
                   * **sourcelines** (*list*) -- a list of all lines in the file. We assume
                     the docstring exists as a pure string literal in the source.
                     In other words, no postprocessing via split, format, or any
                     other dynamic programatic modification should be made to the
                     docstrings. Python's docstring extractor assumes this as well.
                   * **endpos** (*int*) -- line position (starting at 0) the docstring ends on.
                     Note: positions are 0 based but linenos are 1 based.

      :returns:

                start, stop:
                    start: the line position (0 based) the docstring starts on
                    stop: the line position (0 based) that the docstring stops

                    such that sourcelines[start:stop] will contain the docstring
      :rtype: tuple[Int, Int]

      CommandLine:
          python -m xdoctest xdoctest.static_analysis TopLevelVisitor._find_docstr_startpos_workaround
          python -m xdoctest xdoctest.static_analysis TopLevelVisitor._find_docstr_startpos_workaround --debug

      .. rubric:: Example

      >>> # xdoctest: +REQUIRES(CPython)
      >>> # This function is a specific workaround for a CPython bug.
      >>> from xdoctest.static_analysis import *
      >>> sys.DEBUG = '--debug' in sys.argv
      >>> sq = chr(39)  # single quote
      >>> dq = chr(34)  # double quote
      >>> source = utils.codeblock(
          '''
          def func0():
              {ddd} docstr0 {ddd}
          def func1():
              {ddd}
              docstr1 {ddd}
          def func2():
              {ddd} docstr2
              {ddd}
          def func3():
              {ddd}
              docstr3
              {ddd}  # foobar
          def func5():
              {ddd}pathological case
              {sss} # {ddd} # {sss} # {ddd} # {ddd}
          def func6():
              " single quoted docstr "
          def func7():
              r{ddd}
              raw line
              {ddd}
          ''').format(ddd=dq * 3, sss=sq * 3)
      >>> print(utils.add_line_numbers(utils.highlight_code(source), start=0))
      >>> targets = [
      >>>     (1, 2),
      >>>     (3, 5),
      >>>     (6, 8),
      >>>     (9, 12),
      >>>     (13, 15),
      >>>     (16, 17),
      >>>     (18, 21),
      >>> ]
      >>> self = TopLevelVisitor.parse(source)
      >>> pt = ast.parse(source.encode('utf8'))
      >>> sourcelines = source.splitlines()
      >>> # PYPY docnode.lineno specify the startpos of a docstring not
      >>> # the end.
      >>> print('\n\n====\n\n')
      >>> #for i in [0, 1]:
      >>> for i in range(len(targets)):
      >>>     print('----------')
      >>>     funcnode = pt.body[i]
      >>>     print('funcnode = {!r}'.format(funcnode))
      >>>     docnode = funcnode.body[0]
      >>>     print('funcnode.__dict__ = {!r}'.format(funcnode.__dict__))
      >>>     print('docnode = {!r}'.format(docnode))
      >>>     print('docnode.value = {!r}'.format(docnode.value))
      >>>     print('docnode.value.__dict__ = {!r}'.format(docnode.value.__dict__))
      >>>     print('docnode.value.s = {!r}'.format(docnode.value.s))
      >>>     print('docnode.lineno = {!r}'.format(docnode.lineno))
      >>>     print('docnode.col_offset = {!r}'.format(docnode.col_offset))
      >>>     print('docnode = {!r}'.format(docnode))
      >>>     #import IPython
      >>>     #IPython.embed()
      >>>     docstr = ast.get_docstring(funcnode, clean=False)
      >>>     print('len(docstr) = {}'.format(len(docstr)))
      >>>     endpos = docnode.lineno - 1
      >>>     if hasattr(docnode, 'end_lineno'):
      >>>         endpos = docnode.end_lineno - 1
      >>>     print('endpos = {!r}'.format(endpos))
      >>>     start, end = self._find_docstr_startpos_workaround(docstr, sourcelines, endpos)
      >>>     print('i = {!r}'.format(i))
      >>>     print('got  = {}, {}'.format(start, end))
      >>>     print('want = {}, {}'.format(*targets[i]))
      >>>     if targets[i] != (start, end):
      >>>         print('---')
      >>>         print(docstr)
      >>>         print('---')
      >>>         print('sourcelines = [\n{}\n]'.format(', \n'.join(list(map(repr, enumerate(sourcelines))))))
      >>>         print('endpos = {!r}'.format(endpos))
      >>>         raise AssertionError('docstr workaround is failing')
      >>>     print('----------')
      >>> sys.DEBUG = 0


   .. method:: _get_docstring(self, node)


      CommandLine:
          xdoctest -m ~/code/xdoctest/xdoctest/static_analysis.py TopLevelVisitor._get_docstring

      .. rubric:: Example

      >>> source = utils.codeblock(
          '''
          def foo():
              'docstr'
          ''')
      >>> self = TopLevelVisitor(source)
      >>> node = self.syntax_tree().body[0]
      >>> self._get_docstring(node)
      ('docstr', 2, 2)


   .. method:: _workaround_func_lineno(self, node)


      Finds the correct line for the original function definition even when
      decorators are involved.

      .. rubric:: Example

      >>> source = utils.codeblock(
          '''
          @bar
          @baz
          def foo():
              'docstr'
          ''')
      >>> self = TopLevelVisitor(source)
      >>> node = self.syntax_tree().body[0]
      >>> self._workaround_func_lineno(node)
      3



.. function:: parse_static_calldefs(source=None, fpath=None)

   Statically finds top-level callable functions and methods in python source

   :Parameters: * **source** (*str*) -- python text
                * **fpath** (*str*) -- filepath to read if source is not specified

   :returns:

                 maping from callnames to CallDefNodes, which contain
                    info about the item with the doctest.
   :rtype: Dict[str, CallDefNode]

   .. rubric:: Example

   >>> from xdoctest import static_analysis
   >>> fpath = static_analysis.__file__.replace('.pyc', '.py')
   >>> calldefs = parse_static_calldefs(fpath=fpath)
   >>> assert 'parse_static_calldefs' in calldefs


.. function:: parse_calldefs(source=None, fpath=None)


.. function:: _parse_static_node_value(node)

   Extract a constant value from a node if possible


.. function:: parse_static_value(key, source=None, fpath=None)

   Statically parse a constant variable's value from python code.

   TODO: This does not belong here. Move this to an external static analysis
   library.

   :Parameters: * **key** (*str*) -- name of the variable
                * **source** (*str*) -- python text
                * **fpath** (*str*) -- filepath to read if source is not specified

   .. rubric:: Example

   >>> from xdoctest.static_analysis import parse_static_value
   >>> key = 'foo'
   >>> source = 'foo = 123'
   >>> assert parse_static_value(key, source=source) == 123
   >>> source = 'foo = "123"'
   >>> assert parse_static_value(key, source=source) == '123'
   >>> source = 'foo = [1, 2, 3]'
   >>> assert parse_static_value(key, source=source) == [1, 2, 3]
   >>> source = 'foo = (1, 2, "3")'
   >>> assert parse_static_value(key, source=source) == (1, 2, "3")
   >>> source = 'foo = {1: 2, 3: 4}'
   >>> assert parse_static_value(key, source=source) == {1: 2, 3: 4}
   >>> source = 'foo = None'
   >>> assert parse_static_value(key, source=source) == None
   >>> #parse_static_value('bar', source=source)
   >>> #parse_static_value('bar', source='foo=1; bar = [1, foo]')


.. function:: package_modpaths(pkgpath, with_pkg=False, with_mod=True, followlinks=True, recursive=True, with_libs=False, check=True)

   Finds sub-packages and sub-modules belonging to a package.

   :Parameters: * **pkgpath** (*str*) -- path to a module or package
                * **with_pkg** (*bool*) -- if True includes package __init__ files (default =
                  False)
                * **with_mod** (*bool*) -- if True includes module files (default = True)
                * **exclude** (*list*) -- ignores any module that matches any of these patterns
                * **recursive** (*bool*) -- if False, then only child modules are included
                * **with_libs** (*bool*) -- if True then compiled shared libs will be returned as well
                * **check** (*bool*) -- if False, then then pkgpath is considered a module even
                  if it does not contain an __init__ file.

   :Yields: *str* -- module names belonging to the package

   .. rubric:: References

   http://stackoverflow.com/questions/1707709/list-modules-in-py-package

   .. rubric:: Example

   >>> from xdoctest.static_analysis import *
   >>> pkgpath = modname_to_modpath('xdoctest')
   >>> paths = list(package_modpaths(pkgpath))
   >>> print('\n'.join(paths))
   >>> names = list(map(modpath_to_modname, paths))
   >>> assert 'xdoctest.core' in names
   >>> assert 'xdoctest.__main__' in names
   >>> assert 'xdoctest' not in names
   >>> print('\n'.join(names))


.. function:: is_balanced_statement(lines, only_tokens=False)

   Checks if the lines have balanced parens, brakets, curlies and strings

   :Parameters: **lines** (*list*) -- list of strings

   :returns: False if the statement is not balanced
   :rtype: bool

   CommandLine:
       xdoctest -m xdoctest.static_analysis is_balanced_statement

   Doctest:
       >>> assert is_balanced_statement(['print(foobar)'])
       >>> assert is_balanced_statement(['foo = bar']) is True
       >>> assert is_balanced_statement(['foo = (']) is False
       >>> assert is_balanced_statement(['foo = (', "')(')"]) is True
       >>> assert is_balanced_statement(
       ...     ['foo = (', "'''", ")]'''", ')']) is True
       >>> assert is_balanced_statement(
       ...     ['foo = ', "'''", ")]'''", ')']) is False
       >>> #assert is_balanced_statement(['foo = ']) is False
       >>> #assert is_balanced_statement(['== ']) is False
       >>> lines = ['def foo():', '', '    x = 1', 'assert True', '']
       >>> assert is_balanced_statement(lines)

   Doctest:
       >>> from xdoctest.static_analysis import *
       >>> source_parts = [
       >>>     'setup(',
       >>>     "    name='extension',",
       >>>     '    ext_modules=[',
       >>>     '        CppExtension(',
       >>>     "            name='extension',",
       >>>     "            sources=['extension.cpp'],",
       >>>     "            extra_compile_args=['-g'])),",
       >>>     '    ],',
       >>> ]
       >>> print('\n'.join(source_parts))
       >>> assert not is_balanced_statement(source_parts)
       >>> source_parts = [
       >>>     'setup(',
       >>>     "    name='extension',",
       >>>     '    ext_modules=[',
       >>>     '        CppExtension(',
       >>>     "            name='extension',",
       >>>     "            sources=['extension.cpp'],",
       >>>     "            extra_compile_args=['-g']),",
       >>>     '    ],',
       >>>     '        cmdclass={',
       >>>     "            'build_ext': BuildExtension",
       >>>     '        })',
       >>> ]
       >>> print('\n'.join(source_parts))
       >>> assert is_balanced_statement(source_parts)


.. function:: extract_comments(source)

   Returns the text in each comment in a block of python code.
   Uses tokenize to account for quotations.

   CommandLine:
       python -m xdoctest.static_analysis extract_comments

   .. rubric:: Example

   >>> from xdoctest import utils
   >>> source = utils.codeblock(
   >>>    '''
          # comment 1
          a = '# not a comment'  # comment 2
          c = 3
          ''')
   >>> comments = list(extract_comments(source))
   >>> assert comments == ['# comment 1', '# comment 2']
   >>> comments = list(extract_comments(source.splitlines()))
   >>> assert comments == ['# comment 1', '# comment 2']


.. function:: six_axt_parse(source_block, filename='<source_block>', compatible=True)

   Python 2/3 compatible replacement for ast.parse(source_block, filename='<source_block>')


